# Grafana Alloy Helm values for OSDC.
#
# Alloy pushes metrics to Grafana Cloud by independently discovering
# ServiceMonitor/PodMonitor CRDs and scraping the same targets as Prometheus.
# Only installed when the grafana-cloud-credentials secret exists.
#
# Dynamic values (GCLOUD_RW_URL, CLUSTER_NAME) are injected by deploy.sh
# via a temporary override file.

controller:
  type: deployment
  replicas: 2

  # Run on base infrastructure nodes
  nodeSelector:
    role: base-infrastructure
  tolerations:
    - key: CriticalAddonsOnly
      operator: Equal
      value: "true"
      effect: NoSchedule

alloy:
  clustering:
    enabled: true
  resources:
    requests:
      cpu: 500m
      memory: 2Gi
    limits:
      cpu: "4"
      memory: 8Gi
  configMap:
    content: |-
      // Discover and scrape all ServiceMonitor targets across namespaces.
      prometheus.operator.servicemonitors "default" {
        forward_to = [prometheus.relabel.cost_control.receiver]
        clustering {
          enabled = true
        }
      }

      // Discover and scrape all PodMonitor targets across namespaces.
      prometheus.operator.podmonitors "default" {
        forward_to = [prometheus.relabel.cost_control.receiver]
        clustering {
          enabled = true
        }
      }

      // Drop high-cardinality and low-value metrics before remote_write.
      // Most filtering is done at ServiceMonitor level (keep whitelists).
      // These rules catch anything that slips through from sources without per-monitor filtering.
      prometheus.relabel "cost_control" {
        forward_to = [prometheus.remote_write.grafana_cloud.receiver]

        // --- KSM: drop low-value metrics ---
        rule {
          action        = "drop"
          source_labels = ["__name__"]
          regex         = "kube_.*_created|kube_.*_metadata_resource_version|kube_secret_.*|kube_configmap_.*|kube_endpoint_.*|kube_lease_.*"
        }

        // --- keep restarts_total and cadvisor memory only for control-plane namespaces ---
        // These metrics generate one series per container, so storing them cluster-wide
        // (especially with high runner pod churn) is too expensive. We only need them for
        // control-plane namespaces where we actually alert (e.g. ControlPlaneCrashLoop).
        //
        // Two-step replace+drop pattern (RE2 has no lookahead):
        //   1. Tag matching metrics in control-plane namespaces with __keep_cp__="true"
        //   2. Drop any matching metric that was NOT tagged (i.e. __keep_cp__ is empty)
        rule {
          action        = "replace"
          source_labels = ["__name__", "namespace"]
          regex         = "(kube_pod_container_status_restarts_total|container_memory_working_set_bytes|container_memory_rss);(arc-systems|karpenter|harbor-system|monitoring|logging|buildkit)"
          target_label  = "__keep_cp__"
          replacement   = "true"
        }
        // Also keep restarts_total for the bin-pack-scheduler pods specifically.
        // The secondary scheduler in kube-system OOMKills (exit 137) under memory
        // pressure; retaining its restart counter lets us count OOM events over
        // time (increase() over the counter) instead of only seeing the current
        // pod's last-termination reason gauge, which resets on every redeploy.
        // Scoped to the pod-name prefix so we do NOT retain restart series for
        // every per-node kube-system DaemonSet pod (that would be a cardinality /
        // billing blowup — the whole reason kube-system is excluded above).
        rule {
          action        = "replace"
          source_labels = ["__name__", "namespace", "pod"]
          regex         = "kube_pod_container_status_restarts_total;kube-system;bin-pack-scheduler-.*"
          target_label  = "__keep_cp__"
          replacement   = "true"
        }
        rule {
          action        = "drop"
          source_labels = ["__name__", "__keep_cp__"]
          regex         = "(kube_pod_container_status_restarts_total|container_memory_working_set_bytes|container_memory_rss);"
        }
        // Clean up temporary label before remote_write
        rule {
          action       = "labeldrop"
          regex        = "__keep_cp__"
        }

        // --- drop misc high-cardinality metrics ---
        rule {
          action        = "drop"
          source_labels = ["__name__"]
          regex         = "kubernetes_feature_enabled"
        }

        // --- ARC histogram buckets (keep sum/count) ---
        rule {
          action        = "drop"
          source_labels = ["__name__"]
          regex         = "gha_job_(execution|startup)_duration_seconds_bucket"
        }

        // --- go/process/promhttp/prometheus_operator internals from any source ---
        rule {
          action        = "drop"
          source_labels = ["__name__"]
          regex         = "go_.*|process_.*|promhttp_.*|prometheus_operator_.*"
        }
      }

      // Sync PrometheusRule CRDs to Grafana Cloud Managed Alerts.
      // Without this, PrometheusRule CRDs are inert (no local Prometheus).
      mimir.rules.kubernetes "grafana_cloud" {
        address  = env("GCLOUD_BASE_URL")
        tenant_id = env("GCLOUD_RW_API_USER")
        mimir_namespace_prefix = env("CLUSTER_NAME")

        basic_auth {
          username = env("GCLOUD_RW_API_USER")
          password = env("GCLOUD_RW_API_KEY")
        }
      }

      // Push metrics to Grafana Cloud Mimir.
      prometheus.remote_write "grafana_cloud" {
        external_labels = {
          cluster = env("CLUSTER_NAME"),
        }
        endpoint {
          url = env("GCLOUD_RW_URL")
          basic_auth {
            username = env("GCLOUD_RW_API_USER")
            password = env("GCLOUD_RW_API_KEY")
          }
        }
      }
