apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: network-pressure-alerts
  namespace: monitoring
  labels:
    app.kubernetes.io/part-of: osdc-monitoring
spec:
  groups:
    - name: ipv6-network-pressure
      rules:
        - alert: NodeConntrackHighWarn
          expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.80
          for: 5m
          labels:
            severity: warning
            team: pytorch-dev-infra
            priority: P3
          annotations:
            summary: "Node conntrack table > 80% full ({{ $labels.node }})"
            description: |
              Node {{ $labels.node }} conntrack table is {{ $value | humanizePercentage }} full.
              Under IPv6-only EKS, pod-to-IPv4-internet egress goes via VPC CNI's IPv4 SNAT,
              which is conntrack-heavy. Pre-cutover baseline should be well below this threshold.
              Investigate via `ssh <node> sudo conntrack -L | wc -l`.

        - alert: NodeConntrackHighCritical
          expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.95
          for: 2m
          labels:
            severity: critical
            team: pytorch-dev-infra
            priority: P3
          annotations:
            summary: "Node conntrack table > 95% full — imminent connection drops ({{ $labels.node }})"
            description: |
              Node {{ $labels.node }} conntrack table is {{ $value | humanizePercentage }} full.
              New connections will start failing imminently. Increase `nf_conntrack_max` sysctl
              OR drain the node. See docs/ipv6-cluster-recreation.md for tuning guidance.

        - alert: NodeConntrackMetricMissing
          # Detect silent loss of node-exporter scraping. Without this, the
          # NodeConntrackHigh{Warn,Critical} alerts above would never fire if
          # node-exporter scraping breaks (e.g. IPv6 endpoint regression),
          # and the operator would only learn from empty dashboards.
          expr: absent(node_nf_conntrack_entries)
          for: 10m
          labels:
            severity: warning
            team: pytorch-dev-infra
            priority: P3
          annotations:
            summary: "node_nf_conntrack_entries metric is missing — conntrack alerts inert"
            description: |
              The node-exporter conntrack metric has been absent for 10m. Either
              all nodes are down (unlikely) or node-exporter scraping is broken.
              The NodeConntrackHigh alerts cannot fire while this is true.
              Verify Alloy → node-exporter scrape: `up{job=~".*node-exporter.*"}`.
