Grafana

cd /home/myusername/docker

mkdir -p grafana/{prometheus/data,loki,tempo,minio/buckets/tempo,minio/buckets/loki-data,minio/buckets/loki-ruler,alloy/config,alloy/data,data,datasources} && cd grafana

nano docker-compose.yml

services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    hostname: prometheus
    user: root
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=240h'
      - '--web.enable-lifecycle'
      - '--web.enable-remote-write-receiver' 
    volumes:
      - ./prometheus/data:/prometheus
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - 8320:9090
    labels:
      org.label-schema.group: "monitoring"
    networks:
      - management
    depends_on:
      - alloy

  # THE UNIFIED COLLECTOR (Replaces NodeExporter, cAdvisor, FluentBit)
  alloy:
    image: grafana/alloy:latest
    container_name: alloy
    hostname: alloy
    privileged: true # Required for cAdvisor/NodeExporter functionality
    pid: host        # Required to see host processes
    command:
      - run
      - --server.http.listen-addr=0.0.0.0:12345
      - --storage.path=/var/lib/alloy/data
      - /etc/alloy/config.alloy
    volumes:
      - ./alloy/config/config.alloy:/etc/alloy/config.alloy
      - ./alloy/data:/var/lib/alloy/data
      # Mounts required for Host Metrics (Node Exporter replacement)
      - /:/rootfs:ro
      - /sys:/host/sys:ro
      - /proc:/host/proc:ro
      # Mounts required for Container Logs
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/log:/var/log:ro
      # Mounts required for Container Metrics (cAdvisor replacement)
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /run/udev/data:/run/udev/data:ro
    ports:
      - 8325:12345 # Alloy WEBUI
      - 4317:4317  # OTel gRPC
      - 4318:4318  # OTel HTTP
    networks:
      - management
    restart: unless-stopped

# LOG COLLECTOR
  loki:
    image: grafana/loki:latest
    container_name: loki
    hostname: loki
    ports:
      - 8324:3100
    volumes:
      - ./loki/:/etc/loki/
    command: -config.file=/etc/loki/config.yml
    networks:
      - management

# TEMPORARY STORAGE BACKEND FOR LOKI & TEMPO
  minio:
    image: minio/minio:latest
    container_name: minio
    hostname: minio
    user: root
    environment:
      MINIO_ROOT_USER: tempo
      MINIO_ROOT_PASSWORD: supersecret
    volumes:
      - ./minio/buckets:/data
    command: server /data --console-address ':9001'
    networks:
      - management

# TRACING BACKEND FOR GRAFANA
  tempo:
    image: grafana/tempo:3.0.0
    container_name: tempo
    hostname: tempo
    command: "-target=all -config.file=/etc/tempo.yaml"
    volumes:
      - ./tempo/tempo.yml:/etc/tempo.yaml
    ports:
      - 8327:3200
    networks:
      - management
    depends_on:
      - minio

# METRICS MAIN GUI DASHBOARD AND VISUALIZER
  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    hostname: grafana
    user: root
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_USERS_ALLOW_SIGN_UP: false
      GF_SMTP_ENABLED: true
      GF_SMTP_HOST: mail.DOMAIN.COM:587
      GF_SMTP_USER: [email protected]
      GF_SMTP_PASSWORD: INSERT_PASSWORD_HERE
      GF_FEATURE_TOGGLES_ENABLE: traceqlEditor
    volumes:
      - ./data:/var/lib/grafana
      - ./datasources:/etc/grafana/provisioning/datasources
    ports:
      - 8321:3000
    labels:
      org.label-schema.group: "monitoring"
    restart: unless-stopped
    networks:
      - management

networks:
  management:
    external: true

nano datasources/datasources.yml

apiVersion: 1

datasources:
  # 1. PROMETHEUS (Loads first, no dependencies)
  - name: Prometheus
    type: prometheus
    uid: prometheus
    access: proxy
    url: http://prometheus:9090
    basicAuth: false
    isDefault: false
    version: 1
    editable: true
    jsonData:
      httpMethod: POST

  # 2. LOKI
  - name: Loki
    type: loki
    uid: loki
    access: proxy
    url: http://loki:3100
    basicAuth: false
    isDefault: false
    version: 1
    editable: true
    jsonData:
      maxLines: 1000
      derivedFields:
        - datasourceUid: tempo
          matcherRegex: "traceId=(\\w+)"
          name: TraceID
          url: "$${__value.raw}"

  # 3. TEMPO (Loads last. Can safely find Prometheus and Loki)
  - name: Tempo
    type: tempo
    uid: tempo
    access: proxy
    url: http://tempo:3200
    basicAuth: false
    isDefault: true
    version: 1
    editable: true
    apiVersion: 1
    jsonData:
      httpMethod: GET
      serviceMap:
        datasourceUid: prometheus
      lokiSearch:
        datasourceUid: loki
      nodeGraph:
        enabled: true

nano prometheus/prometheus.yml

global:
  scrape_interval: 1m
  evaluation_interval: 1m

scrape_configs: [] # Empty, because Alloy does the scraping now!

nano tempo/tempo.yml

stream_over_http_enabled: true

server:
  http_listen_port: 3200

distributor:
  receivers:                           
    otlp:
      protocols:
        http:
        grpc:

memberlist:
  abort_if_cluster_join_fails: false
  bind_port: 7946
  join_members:
  - tempo:7946

metrics_generator:
  registry:
    external_labels:
      source: tempo
      cluster: docker-compose
  storage:
    path: /tmp/tempo/generator/wal
    remote_write:
      - url: http://prometheus:9090/api/v1/write
        send_exemplars: true

storage:
  trace:
    backend: s3                        
    wal:
      path: /tmp/tempo/wal             
    s3:
      bucket: tempo                    
      endpoint: minio:9000
      access_key: tempo
      secret_key: supersecret
      insecure: true
      region: us-west-1                

querier:
  frontend_worker:
    frontend_address: tempo:9095

overrides:
  defaults:
    metrics_generator:
      processors: ['service-graphs', 'span-metrics']

nano loki/config.yml

auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

common:
  path_prefix: /loki
  replication_factor: 1
  storage:
    s3:
      endpoint: minio:9000
      insecure: true
      bucketnames: loki-data
      access_key_id: tempo
      secret_access_key: supersecret
      s3forcepathstyle: true
  ring:
    kvstore:
      store: memberlist

schema_config:
  configs:
    - from: 2024-01-01
      store: tsdb
      object_store: s3
      schema: v13
      index:
        prefix: index_
        period: 24h

storage_config:
  tsdb_shipper:
    active_index_directory: /loki/tsdb-index
    cache_location: /loki/tsdb-cache
    cache_ttl: 24h

limits_config:
  reject_old_samples: false
  reject_old_samples_max_age: 168h
  allow_structured_metadata: true

ruler:
  storage:
    s3:
      bucketnames: loki-ruler
  alertmanager_url: http://localhost:9093

nano alloy/config/config.alloy

// ============================================================================
// 1. SYSTEM & DOCKER METRICS
// ============================================================================

prometheus.exporter.unix "host_metrics" {
  rootfs_path = "/rootfs"
  sysfs_path  = "/host/sys"
  procfs_path = "/host/proc"
  
  filesystem {
    fs_types_exclude     = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
    mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)"
    mount_timeout        = "5s"
  }
}

prometheus.exporter.cadvisor "docker_metrics" {
  docker_host = "unix:///var/run/docker.sock"
}

prometheus.relabel "docker_metrics_filter" {
  forward_to = [prometheus.remote_write.local_prom.receiver]
  rule {
    source_labels = ["__name__"]
    regex = "container_spec_(cpu_period|cpu_quota|cpu_shares|memory_limit_bytes|memory_swap_limit_bytes|memory_reservation_limit_bytes)"
    action = "drop"
  }
}

// ============================================================================
// 2. SCRAPE JOBS
// ============================================================================

prometheus.scrape "scrape_host" {
  targets    = prometheus.exporter.unix.host_metrics.targets
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

prometheus.scrape "scrape_docker" {
  targets    = prometheus.exporter.cadvisor.docker_metrics.targets
  forward_to = [prometheus.relabel.docker_metrics_filter.receiver]
}

prometheus.scrape "scrape_traefik" {
  targets = [{ __address__ = "traefik:8083", job = "traefik" }]
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

prometheus.scrape "scrape_immich" {
  targets = [
    { __address__ = "immich-server:8081", job = "immich_server" },
    { __address__ = "immich-microservices:8081", job = "immich_microservices" }
  ]
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

prometheus.scrape "scrape_pihole" {
  targets = [{ __address__ = "piholeexporter:9617", job = "metrics_pihole_exporter" }]
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

prometheus.scrape "scrape_librespeed" {
  targets = [{ __address__ = "librespeedexporter:9469", job = "metrics_librespeed_exporter" }]
  metrics_path = "/probe"
  params = { "script" = ["librespeed"] }
  scrape_interval = "360m"
  scrape_timeout  = "2m"
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

prometheus.scrape "scrape_crowdsec" {
  targets = [{ __address__ = "YOURSERVERIP:6060", job = "crowdsec" }]
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

prometheus.scrape "scrape_infra" {
  targets = [
    { __address__ = "loki:3100",  job = "loki" },
    { __address__ = "tempo:3200", job = "tempo" },
  ]
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

// ============================================================================
// 3. LOG COLLECTION
// ============================================================================

discovery.docker "docker_logs" {
  host = "unix:///var/run/docker.sock"
}

discovery.relabel "docker_logs_relabel" {
  targets = discovery.docker.docker_logs.targets
  rule {
    source_labels = ["__meta_docker_container_name"]
    regex         = "/(.*)"
    target_label  = "service_name"
  }
}

loki.source.docker "collect_docker_logs" {
  host       = "unix:///var/run/docker.sock"
  targets    = discovery.relabel.docker_logs_relabel.output
  forward_to = [loki.process.process_logs.receiver]
}

loki.process "process_logs" {
  forward_to = [loki.write.local_loki.receiver]
  stage.match {
    selector = "{service_name=\"traefik\"}"
    stage.json {
      expressions = { msg = "log" }
    }
    stage.output { source = "msg" }
  }
}

// ============================================================================
// 4. TRACES (OTel)
// ============================================================================

otelcol.receiver.otlp "default" {
  http { endpoint = "0.0.0.0:4318" }
  grpc { endpoint = "0.0.0.0:4317" }

  output {
    traces  = [otelcol.exporter.otlp.tempo.input]
    metrics = [otelcol.exporter.prometheus.otel_to_prom.input] 
    logs    = [otelcol.exporter.loki.otel_to_loki.input]
  }
}

otelcol.exporter.prometheus "otel_to_prom" {
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

otelcol.exporter.loki "otel_to_loki" {
  forward_to = [loki.write.local_loki.receiver]
}

otelcol.exporter.otlp "tempo" {
  client {
    endpoint = "tempo:4317"
    tls { insecure = true }
  }
}

// ============================================================================
// 5. OUTPUTS
// ============================================================================

prometheus.remote_write "local_prom" {
  endpoint { url = "http://prometheus:9090/api/v1/write" }
}

loki.write "local_loki" {
  endpoint { url = "http://loki:3100/loki/api/v1/push" }
}

Docker Logging Configuration

WARNING: The loki-docker-driver plugin is deprecated and will break Alloy log collection. We must revert Docker to standard JSON logging so Alloy can read the log files natively.

sudo nano /etc/docker/daemon.json

{
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "10m",
        "max-file": "3"
    }
}

• With Nvidia Runtime:

{
  "default-runtime": "nvidia",
  "runtimes": {
    "nvidia": {
      "path": "/usr/bin/nvidia-container-runtime",
      "runtimeArgs": []
    }
  },
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  }
}

sudo systemctl restart docker
docker compose up -d

(Note: Any containers that existed prior to changing the logging driver must be recreated (docker compose down && docker compose up -d) to inherit the new JSON logger.)

Configure Grafana

Default login
Username: admin
Password: admin

1. Go to: Home -> Connections -> Data sources
2. Your data sources (Prometheus, Loki, Tempo) are auto-provisioned via the datasources.yml config!
3. Download and import dashboards at: Home -> Dashboards -> Import dashboard

• docker_containers.json
• node-exporter-full.json
• monitoring_services.json - Optional

Configure Authentik SSO (Generic OAuth) & Role Mapping

By using Authentik, you can manage Grafana access and role assignment centrally. Users will automatically be assigned Admin, Editor, or Viewer roles based on Authentik entitlements.

1. Authentik Setup

1. In the Authentik Admin Dashboard, go to Applications > Applications and create a new Application & Provider pair.
2. Provider Type: OAuth2/OpenID Connect
3. Set the Strict Redirect URI exactly to your Grafana URL appended with /login/generic_oauth (e.g., https://stats.DOMAIN.COM/login/generic_oauth).
4. Under Advanced Protocol Settings, ensure the scopes openid, profile, email, and entitlements are selected.
5. Take note of the Client ID and Client Secret.

2. Configure Entitlements in Authentik (Role Mapping)

Grafana will read Authentik "entitlements" to decide a user's permission level.

1. In Authentik, go to Applications > Applications and click your newly created Grafana application.
2. Go to the Application entitlements tab.
3. Click Create:
   • Title: Type exactly Grafana Admins
   • Attributes: Leave this completely blank!
   • Click Create, then bind your Admin users/groups to this entitlement.
4. Click Create again:
   • Title: Type exactly Grafana Editors
   • Attributes: Leave this completely blank!
   • Click Create, then bind your Editor users/groups to this entitlement.

Note: You do not need to create a Viewer entitlement. Anyone who logs in without the Admin or Editor entitlement will automatically fall back to the Viewer role safely!

3. Grafana UI Generic OAuth Settings

Log in to Grafana with the local admin account and navigate to Administration > Authentication > Generic OAuth.
Fill out the fields exactly as follows:

• Client ID: (Paste from Authentik)
• Client Secret: (Paste from Authentik)
• Scopes: openid profile email entitlements
• Auth URL: https://authentik.DOMAIN.COM/application/o/authorize/
• Token URL: https://authentik.DOMAIN.COM/application/o/token/
• API URL: https://authentik.DOMAIN.COM/application/o/userinfo/
• Allow Sign Up: Enabled
• Sign out redirect URL: https://authentik.DOMAIN.COM/application/o/<app-slug>/end-session/
• Role Attribute Path: (Paste the code below exactly)

  contains(entitlements[*], 'Grafana Admins') && 'Admin' || contains(entitlements[*], 'Grafana Editors') && 'Editor' || 'Viewer'
  

• Use PKCE: Enabled

Save the configuration. If you ever need to bypass SSO to reach the local login screen, navigate to https://stats.DOMAIN.COM/login?forceLogin=true.

Adding External Services to Grafana (Crowdsec, Pi-Hole, Librespeed)

Since Prometheus no longer handles scraping directly, DO NOT edit prometheus.yml.
All new scraping targets must be added to Alloy.

Add Crowdsec dashboard

Make sure you have port 6060 open at the crowdsec docker container.
The scrape configuration for Crowdsec is already included in your alloy/config/config.alloy file!

1. Import Dashboard: https://github.com/crowdsecurity/grafana-dashboards

Add Librespeed dashboard

cd /home/myusername/docker
mkdir librespeed_exporter && cd "$_"

nano docker-compose.yml

services:
  librespeed_exporter:
    image: brendonmatheson/prometheus-librespeed-exporter:1.0.0
    container_name: librespeed-exporter
    restart: unless-stopped
    hostname: librespeedexporter
    ports:
      - 8162:9469

docker compose up -d

(The scrape config for Librespeed is already active in your alloy/config/config.alloy file!)

Add Pi-hole dashboard

cd /home/myusername/docker/pihole
mkdir pihole_exporter && cd "$_"

nano docker-compose.yml

services:
  pihole-exporter:
    image: ekofr/pihole-exporter:latest
    container_name: pihole-exporter
    hostname: piholeexporter
    environment:
      PIHOLE_PROTOCOL: http
      PIHOLE_HOSTNAME: pihole01
      PIHOLE_PASSWORD: pihole
      PIHOLE_PORT: 80
      PORT: 9617
    ports:
      - 8125:9617

docker compose up -d

(The scrape config for Pi-Hole is already active in your alloy/config/config.alloy file!)

AUTOMATION SCRIPT

A bash script you can run on a fresh server to automatically create the folders, write the config files, configure the Docker daemon, and spin everything up.

nano install_grafana_stack.sh

#!/bin/bash
set -e

echo "==============================================="
echo " Installing The Grafana Stack"
echo "==============================================="

# Set base directory
BASE_DIR="$HOME/docker/grafana"

# Create folders
mkdir -p "$BASE_DIR"/{prometheus/data,loki,tempo,minio/buckets/tempo,minio/buckets/loki-data,minio/buckets/loki-ruler,alloy/config,alloy/data,data,datasources}
cd "$BASE_DIR"

# Ensure Management Network exists
docker network inspect management >/dev/null 2>&1 || docker network create management

echo "[1/6] Creating docker-compose.yml..."
cat << 'EOF' > docker-compose.yml
services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    hostname: prometheus
    user: root
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=240h'
      - '--web.enable-lifecycle'
      - '--web.enable-remote-write-receiver' 
    volumes:
      - ./prometheus/data:/prometheus
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - 8320:9090
    networks:
      - management
    depends_on:
      - alloy

  alloy:
    image: grafana/alloy:latest
    container_name: alloy
    hostname: alloy
    privileged: true 
    pid: host        
    command:
      - run
      - --server.http.listen-addr=0.0.0.0:12345
      - --storage.path=/var/lib/alloy/data
      - /etc/alloy/config.alloy
    volumes:
      - ./alloy/config/config.alloy:/etc/alloy/config.alloy
      - ./alloy/data:/var/lib/alloy/data
      - /:/rootfs:ro
      - /sys:/host/sys:ro
      - /proc:/host/proc:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/log:/var/log:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /run/udev/data:/run/udev/data:ro
    ports:
      - 8325:12345 
      - 4317:4317  
      - 4318:4318  
    networks:
      - management
    restart: unless-stopped

  loki:
    image: grafana/loki:latest
    container_name: loki
    hostname: loki
    ports:
      - 8324:3100
    volumes:
      - ./loki/:/etc/loki/
    command: -config.file=/etc/loki/config.yml
    networks:
      - management

  minio:
    image: minio/minio:latest
    container_name: minio
    hostname: minio
    user: root
    environment:
      MINIO_ROOT_USER: tempo
      MINIO_ROOT_PASSWORD: supersecret
    volumes:
      - ./minio/buckets:/data
    command: server /data --console-address ':9001'
    networks:
      - management

  tempo:
    image: grafana/tempo:3.0.0
    container_name: tempo
    hostname: tempo
    command: "-target=all -config.file=/etc/tempo.yaml"
    volumes:
      - ./tempo/tempo.yml:/etc/tempo.yaml
    ports:
      - 8327:3200
    networks:
      - management
    depends_on:
      - minio

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    hostname: grafana
    user: root
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_USERS_ALLOW_SIGN_UP: false
      GF_FEATURE_TOGGLES_ENABLE: traceqlEditor
    volumes:
      - ./data:/var/lib/grafana
      - ./datasources:/etc/grafana/provisioning/datasources
    ports:
      - 8321:3000
    restart: unless-stopped
    networks:
      - management

networks:
  management:
    external: true
EOF

echo "[2/6] Configuring Prometheus & Datasources..."
cat << 'EOF' > prometheus/prometheus.yml
global:
  scrape_interval: 1m
  evaluation_interval: 1m
scrape_configs: [] 
EOF

cat << 'EOF' > datasources/datasources.yml
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    uid: prometheus
    access: proxy
    url: http://prometheus:9090
    basicAuth: false
    isDefault: false
    version: 1
    editable: true
    jsonData:
      httpMethod: POST

  - name: Loki
    type: loki
    uid: loki
    access: proxy
    url: http://loki:3100
    basicAuth: false
    isDefault: false
    version: 1
    editable: true
    jsonData:
      maxLines: 1000
      derivedFields:
        - datasourceUid: tempo
          matcherRegex: "traceId=(\\w+)"
          name: TraceID
          url: "$${__value.raw}"

  - name: Tempo
    type: tempo
    uid: tempo
    access: proxy
    url: http://tempo:3200
    basicAuth: false
    isDefault: true
    version: 1
    editable: true
    apiVersion: 1
    jsonData:
      httpMethod: GET
      serviceMap:
        datasourceUid: prometheus
      lokiSearch:
        datasourceUid: loki
      nodeGraph:
        enabled: true
EOF

echo "[3/6] Configuring Tempo 3.0 & Loki..."
cat << 'EOF' > tempo/tempo.yml
stream_over_http_enabled: true
server:
  http_listen_port: 3200
distributor:
  receivers:                           
    otlp:
      protocols:
        http:
        grpc:
memberlist:
  abort_if_cluster_join_fails: false
  bind_port: 7946
  join_members:
  - tempo:7946
metrics_generator:
  registry:
    external_labels:
      source: tempo
      cluster: docker-compose
  storage:
    path: /tmp/tempo/generator/wal
    remote_write:
      - url: http://prometheus:9090/api/v1/write
        send_exemplars: true
storage:
  trace:
    backend: s3                        
    wal:
      path: /tmp/tempo/wal             
    s3:
      bucket: tempo                    
      endpoint: minio:9000
      access_key: tempo
      secret_key: supersecret
      insecure: true
      region: us-west-1                
querier:
  frontend_worker:
    frontend_address: tempo:9095
overrides:
  defaults:
    metrics_generator:
      processors: ['service-graphs', 'span-metrics']
EOF

cat << 'EOF' > loki/config.yml
auth_enabled: false
server:
  http_listen_port: 3100
  grpc_listen_port: 9096
common:
  path_prefix: /loki
  replication_factor: 1
  storage:
    s3:
      endpoint: minio:9000
      insecure: true
      bucketnames: loki-data
      access_key_id: tempo
      secret_access_key: supersecret
      s3forcepathstyle: true
  ring:
    kvstore:
      store: memberlist
schema_config:
  configs:
    - from: 2024-01-01
      store: tsdb
      object_store: s3
      schema: v13
      index:
        prefix: index_
        period: 24h
storage_config:
  tsdb_shipper:
    active_index_directory: /loki/tsdb-index
    cache_location: /loki/tsdb-cache
    cache_ttl: 24h
limits_config:
  reject_old_samples: false
  reject_old_samples_max_age: 168h
  allow_structured_metadata: true
ruler:
  storage:
    s3:
      bucketnames: loki-ruler
  alertmanager_url: http://localhost:9093
EOF

echo "[4/6] Configuring Alloy Unified Collector..."
cat << 'EOF' > alloy/config/config.alloy
prometheus.exporter.unix "host_metrics" {
  rootfs_path = "/rootfs"
  sysfs_path  = "/host/sys"
  procfs_path = "/host/proc"
  filesystem {
    fs_types_exclude     = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
    mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)"
    mount_timeout        = "5s"
  }
}

prometheus.exporter.cadvisor "docker_metrics" {
  docker_host = "unix:///var/run/docker.sock"
}

prometheus.scrape "scrape_host" {
  targets    = prometheus.exporter.unix.host_metrics.targets
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

prometheus.scrape "scrape_docker" {
  targets    = prometheus.exporter.cadvisor.docker_metrics.targets
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

prometheus.scrape "scrape_infra" {
  targets = [
    { __address__ = "loki:3100",  job = "loki" },
    { __address__ = "tempo:3200", job = "tempo" },
  ]
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

discovery.docker "docker_logs" {
  host = "unix:///var/run/docker.sock"
}

discovery.relabel "docker_logs_relabel" {
  targets = discovery.docker.docker_logs.targets
  rule {
    source_labels = ["__meta_docker_container_name"]
    regex         = "/(.*)"
    target_label  = "service_name"
  }
}

loki.source.docker "collect_docker_logs" {
  host       = "unix:///var/run/docker.sock"
  targets    = discovery.relabel.docker_logs_relabel.output
  forward_to = [loki.write.local_loki.receiver]
}

otelcol.receiver.otlp "default" {
  http { endpoint = "0.0.0.0:4318" }
  grpc { endpoint = "0.0.0.0:4317" }
  output {
    traces  = [otelcol.exporter.otlp.tempo.input]
    metrics = [otelcol.exporter.prometheus.otel_to_prom.input] 
    logs    = [otelcol.exporter.loki.otel_to_loki.input]
  }
}

otelcol.exporter.prometheus "otel_to_prom" {
  forward_to = [prometheus.remote_write.local_prom.receiver]
}

otelcol.exporter.loki "otel_to_loki" {
  forward_to = [loki.write.local_loki.receiver]
}

otelcol.exporter.otlp "tempo" {
  client {
    endpoint = "tempo:4317"
    tls { insecure = true }
  }
}

prometheus.remote_write "local_prom" {
  endpoint {
    url = "http://prometheus:9090/api/v1/write"
  }
}

loki.write "local_loki" {
  endpoint {
    url = "http://loki:3100/loki/api/v1/push"
  }
}
EOF

echo "[5/6] Ensuring Docker daemon.json uses json-file logger..."
if [ -f /etc/docker/daemon.json ]; then
  echo "Backing up existing /etc/docker/daemon.json to daemon.json.bak"
  sudo cp /etc/docker/daemon.json /etc/docker/daemon.json.bak
fi

sudo bash -c 'cat << EOF > /etc/docker/daemon.json
{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  }
}
EOF'
sudo systemctl restart docker

echo "[6/6] Starting Grafana Stack..."
docker compose up -d

echo "==============================================="
echo " Installation Complete!"
echo " Grafana is running on http://$(hostname -I | awk '{print $1}'):8321"
echo " (Default Login: admin / admin)"
echo " Alloy UI is running on http://$(hostname -I | awk '{print $1}'):8325"
echo "==============================================="

chmod 777 install_grafana_stack.sh && ./install_grafana_stack.sh