# clusters.yaml - Central cluster configuration
#
# Defines every cluster and which modules are installed on each.
# The justfile reads this (via scripts/cluster-config.py) to drive deployments.
#
# Deploy flow per cluster:
#   1. base     — tofu apply (VPC, EKS, Harbor S3/IAM)
#                 mirror bootstrap images, deploy Harbor, base k8s resources
#   2. modules  — each module deployed in listed order
#
# Module deploy contract:
#   Each module may have:
#     terraform/  — independent tofu root (applied separately, gets cluster outputs as -var)
#     kubernetes/ — manifests applied via kubectl/kustomize
#     helm/       — values files for external OCI charts
#     scripts/    — deploy helpers
#
# The base always includes: VPC, EKS, Harbor,
# base k8s resources, NVIDIA device plugin, docker images.
# Harbor is NOT optional — every cluster gets a pull-through cache.

defaults:
  # --- Terraform / base infra ---
  base_node_count: 6
  base_node_instance_type: m7i.12xlarge
  base_node_max_unavailable_percentage: 33
  # TODO(CVE-2026-31431): when bumping to a kernel 6.12.85+ AL2023 AMI,
  # remove osdc/base/kubernetes/algif-mitigation.yaml.
  # https://explore.alas.aws.amazon.com/CVE-2026-31431.html
  # TODO(CVE-2026-43284): when bumping to a kernel with the DirtyFrag fix
  # (6.1.170+ or 6.12.83+ on AL2023), remove
  # osdc/base/kubernetes/dirtyfrag-mitigation.yaml.
  # https://aws.amazon.com/security/security-bulletins/2026-027-aws/
  base_node_ami_version: "v20260318"
  eks_version: "1.35"
  single_nat_gateway: false

  # --- CoreDNS sizing ---
  coredns:
    replicas: 6

  # --- Base component tuning ---
  harbor:
    core_replicas: 4
    registry_replicas: 4
    nginx_replicas: 6
    pdb_max_unavailable: 1  # conservative: at most 1 pod down per component

  # --- Module defaults ---
  karpenter:
    replicas: 4
    log_level: info
    pdb_enabled: true
    pdb_min_available: 1
  arc:
    chart_version: "0.14.1-jeanschmidt.18"  # MUST match for controller + runner charts (minor version mismatch = ARS deletion). Fork chart published from jeanschmidt/actions-runner-controller.
    replica_count: 4
    log_level: info
    controller_cpu_request: "1"
    controller_cpu_limit: "4"
    controller_memory_request: "2Gi"
    controller_memory_limit: "4Gi"
  node_compactor:
    enabled: true
    interval_seconds: 7
    max_uptime_hours: 48
    min_nodes: 1
    dry_run: false
  bin_pack_scheduler:
    replicas: 3
  nodepools:
    gpu_consolidate_after: "10m"
    cpu_consolidate_after: "5m"
    baremetal_consolidate_after: "15m"
    gpu_disruption_budget: "20%"
    cpu_disruption_budget: "20%"
  buildkit:
    replicas_per_arch: 12
    autoscaling:
      enabled: false
  keda:
    chart_version: "2.16.1"
  monitoring:
    grafana_cloud_url: "https://prometheus-prod-36-prod-us-west-0.grafana.net/api/prom/push"
    grafana_cloud_read_url: "https://prometheus-prod-36-prod-us-west-0.grafana.net/api/prom"
  logging:
    grafana_cloud_loki_url: "https://logs-prod-021.grafana.net/loki/api/v1/push"
  alloy_chart_version: "1.6.2"
  zombie_cleanup:
    enabled: true
    pending_max_age_hours: 24
    running_max_age_hours: 24
    busy_max_age_hours: 48
    dry_run: false
  pypi_cache:
    instance_type: r5d.12xlarge
    cuda_versions:
      - "12.6.3"
      - "12.8.1"
      - "13.0.2"
    # NOTE: python_versions is not consumed by manifest generation yet.
    # Reserved for future self-learning builder that will compile wheels
    # for each (cuda_version, python_version) combination.
    python_versions:
      - "3.10"
      - "3.11"
      - "3.12"
      - "3.13"
      - "3.13t"
      - "3.14"
      - "3.14t"
    target_architectures:
      - "x86_64"
      - "aarch64"
    target_manylinux: "2_28"
  hf_cache:
    # Shared HF model cache (modules/hf-cache); enable per cluster via the modules
    # list. Each cluster gets its own bucket pytorch-hf-model-cache-<cluster_id>.
    namespace: hf-cache
    rclone_image: "rclone/rclone:1.69.1"
    # Per-node rclone cache cap; "<N>%" = N% of the node's cache disk, or absolute (200G).
    vfs_cache_max_size: "75%"

clusters:
  meta-staging-aws-uw1:
    region: us-west-1
    cluster_name: meta-staging-aws-uw1
    recycle_karpenter_nodes: true
    proactive_capacity_max: 0  # disable warm-pool pre-provisioning — staging only creates pairs on demand for in-flight jobs
    state_bucket: ciforge-tfstate-meta-staging-aws-uw1
    access_config:
      authentication_mode: API_AND_CONFIG_MAP
      cluster_admin_role_names:
        - osdc_gha_staging
    base:
      base_node_count: 2
      base_node_instance_type: m5.4xlarge
      base_node_max_unavailable_percentage: 100  # all-at-once for staging
      single_nat_gateway: true       # cost optimization for staging
      vpc_cidr: "10.0.0.0/16"
    coredns:
      replicas: 2
    harbor:
      core_replicas: 1
      registry_replicas: 1
      nginx_replicas: 1
      pdb_max_unavailable: "100%"  # aggressive: staging has 1 replica each
    karpenter:
      replicas: 1
      log_level: debug
      pdb_enabled: false
      pdb_min_available: 0
    arc:
      replica_count: 1
      log_level: debug
      controller_cpu_request: "500m"
      controller_cpu_limit: "1"
      controller_memory_request: "1Gi"
      controller_memory_limit: "1Gi"
    node_compactor:
      dry_run: true
    bin_pack_scheduler:
      replicas: 1
    nodepools:
      gpu_consolidate_after: "10s"
      cpu_consolidate_after: "10s"
    arc-runners:
      # Org-wide (was github.com/pytorch/pytorch-canary). A repo-scoped URL forces
      # runner_group → "default" in generate_runners.py, so the org URL is required
      # for the runner_group below to take effect.
      github_config_url: "https://github.com/pytorch"
      github_secret_name: meta-staging-aws-uw1
      runner_name_prefix: "c-mt-"          # staging/testing prefix — keeps labels distinct from prod's "mt-"
      runner_group: "meta-staging-aws-uw1"
      scheduler_name: bin-pack-scheduler
      capacity_aware_fresh_multiplier: 1.0
    buildkit:
      arm64_instance_type: m7gd.16xlarge
      arm64_pods_per_node: 4
      autoscaling:
        enabled: true
        amd64_min: 2  # 1x m6id.24xlarge (2 pods/node)
        amd64_max: 8
        arm64_min: 4  # 1x m7gd.16xlarge (4 pods/node)
        arm64_max: 8
    modules:
      - karpenter
      - arc
      - hf-cache
      - nodepools
      - bin-pack-scheduler
      - arc-runners
      - keda
      - buildkit
      - zombie-cleanup
      - harbor-cache-recovery
      - monitoring
      - logging

  meta-staging-aws-ue1:
    # Second staging region — mirrors meta-staging-aws-uw1 (us-west-1) in us-east-1.
    # Kept long-term alongside uw1 staging for broader region coverage.
    region: us-east-1
    cluster_name: meta-staging-aws-ue1
    recycle_karpenter_nodes: true
    proactive_capacity_max: 0  # disable warm-pool pre-provisioning — staging only creates pairs on demand for in-flight jobs
    state_bucket: ciforge-tfstate-meta-staging-aws-ue1
    access_config:
      authentication_mode: API_AND_CONFIG_MAP
      cluster_admin_role_names:
        - osdc_gha_staging
    base:
      base_node_count: 2
      base_node_instance_type: m5.4xlarge
      base_node_max_unavailable_percentage: 100  # all-at-once for staging
      single_nat_gateway: true       # cost optimization for staging
      vpc_cidr: "10.20.0.0/16"
    coredns:
      replicas: 2
    harbor:
      core_replicas: 1
      registry_replicas: 1
      nginx_replicas: 1
      pdb_max_unavailable: "100%"  # aggressive: staging has 1 replica each
    karpenter:
      replicas: 1
      log_level: debug
      pdb_enabled: false
      pdb_min_available: 0
    arc:
      replica_count: 1
      log_level: debug
      controller_cpu_request: "500m"
      controller_cpu_limit: "1"
      controller_memory_request: "1Gi"
      controller_memory_limit: "1Gi"
    node_compactor:
      dry_run: true
    bin_pack_scheduler:
      replicas: 1
    nodepools:
      gpu_consolidate_after: "10s"
      cpu_consolidate_after: "10s"
    arc-runners:
      # Org-wide (was github.com/pytorch/pytorch-canary). A repo-scoped URL forces
      # runner_group → "default" in generate_runners.py, so the org URL is required
      # for the runner_group below to take effect.
      github_config_url: "https://github.com/pytorch"
      github_secret_name: meta-staging-aws-ue1
      runner_name_prefix: "c-mt-"          # staging/testing prefix — keeps labels distinct from prod's "mt-"
      runner_group: "meta-staging-aws-ue1"
      scheduler_name: bin-pack-scheduler
      capacity_aware_fresh_multiplier: 1.0
    buildkit:
      arm64_instance_type: m7gd.16xlarge
      arm64_pods_per_node: 4
      autoscaling:
        enabled: true
        amd64_min: 2  # 1x m6id.24xlarge (2 pods/node)
        amd64_max: 8
        arm64_min: 4  # 1x m7gd.16xlarge (4 pods/node)
        arm64_max: 8
    pypi_cache:
      replicas: 1
    modules:
      - karpenter
      - arc
      - hf-cache
      - nodepools
      - bin-pack-scheduler
      - arc-runners
      - keda
      - buildkit
      - zombie-cleanup
      - harbor-cache-recovery
      - monitoring
      - logging

  meta-staging-aws-ue2:
    # Third staging region — mirrors meta-staging-aws-ue1 (us-east-1) in us-east-2.
    # Kept long-term alongside uw1/ue1 staging for broader region coverage.
    region: us-east-2
    cluster_name: meta-staging-aws-ue2
    recycle_karpenter_nodes: true
    proactive_capacity_max: 0  # disable warm-pool pre-provisioning — staging only creates pairs on demand for in-flight jobs
    state_bucket: ciforge-tfstate-meta-staging-aws-ue2
    access_config:
      authentication_mode: API_AND_CONFIG_MAP
      cluster_admin_role_names:
        - osdc_gha_staging
    base:
      base_node_count: 2
      base_node_instance_type: m5.4xlarge
      base_node_max_unavailable_percentage: 100  # all-at-once for staging
      single_nat_gateway: true       # cost optimization for staging
      vpc_cidr: "10.24.0.0/16"
    coredns:
      replicas: 2
    harbor:
      core_replicas: 1
      registry_replicas: 1
      nginx_replicas: 1
      pdb_max_unavailable: "100%"  # aggressive: staging has 1 replica each
    karpenter:
      replicas: 1
      log_level: debug
      pdb_enabled: false
      pdb_min_available: 0
    arc:
      replica_count: 1
      log_level: debug
      controller_cpu_request: "500m"
      controller_cpu_limit: "1"
      controller_memory_request: "1Gi"
      controller_memory_limit: "1Gi"
    node_compactor:
      dry_run: true
    bin_pack_scheduler:
      replicas: 1
    nodepools:
      gpu_consolidate_after: "10s"
      cpu_consolidate_after: "10s"
    arc-runners:
      # Org-wide (was github.com/pytorch/pytorch-canary). A repo-scoped URL forces
      # runner_group → "default" in generate_runners.py, so the org URL is required
      # for the runner_group below to take effect.
      github_config_url: "https://github.com/pytorch"
      github_secret_name: meta-staging-aws-ue2
      runner_name_prefix: "c-mt-"          # staging/testing prefix — keeps labels distinct from prod's "mt-"
      runner_group: "meta-staging-aws-ue2"
      scheduler_name: bin-pack-scheduler
      capacity_aware_fresh_multiplier: 1.0
    buildkit:
      arm64_instance_type: m7gd.16xlarge
      arm64_pods_per_node: 4
      autoscaling:
        enabled: true
        amd64_min: 2  # 1x m6id.24xlarge (2 pods/node)
        amd64_max: 8
        arm64_min: 4  # 1x m7gd.16xlarge (4 pods/node)
        arm64_max: 8
    pypi_cache:
      replicas: 1
    modules:
      - karpenter
      - arc
      - hf-cache
      - nodepools
      - bin-pack-scheduler
      - arc-runners
      - keda
      - buildkit
      - zombie-cleanup
      - harbor-cache-recovery
      - monitoring
      - logging

  meta-prod-aws-uw1:
    cluster_name: meta-prod-aws-uw1
    region: us-west-1
    # state_bucket kept on the legacy arc-cbr name to avoid an S3 migration
    # (mirrors meta-prod-aws-ue1, which also kept ciforge-tfstate-arc-cbr-prod-ue1).
    state_bucket: ciforge-tfstate-arc-cbr-prod-uw1
    access_config:
      authentication_mode: API_AND_CONFIG_MAP
      cluster_admin_role_names:
        - osdc_gha_prod
    base:
      vpc_cidr: "10.8.0.0/16"
    node_compactor:
      min_node_age_seconds: 300
    arc-runners:
      github_config_url: "https://github.com/pytorch"
      github_secret_name: meta-prod-aws-uw1
      runner_name_prefix: "mt-"
      runner_group: "meta-prod-aws-uw1"   # distinct from us-east-2's "default" — see doc
      scheduler_name: bin-pack-scheduler
    arc-runners-h100:
      capacity_aware_fresh_multiplier: 1.0
    nodepools-h100:
      # us-west-1 H100 capacity reservations (p5.48xlarge, 8× H100 each):
      #   cr-04d3d1d84e127a562 — 2 nodes (16 H100 GPUs)
      #   cr-09a53051589034fb8 — 4 nodes (32 H100 GPUs)
      # Total: 6 nodes, 48 H100 GPUs.
      # Generator change to honor this override is pending — see
      # docs/prod-cluster-ha-us-west-1.md "Phase 1 prerequisite 2".
      capacity_reservation_ids:
        - cr-04d3d1d84e127a562
        - cr-09a53051589034fb8
    pypi_cache:
      replicas: 10
    modules:
      - karpenter
      - arc
      - hf-cache              # before nodepools: mount must be up before the node-init gate taint
      - nodepools
      - nodepools-h100        # H100 only — B200 has no capacity reservation in us-west-1
      - bin-pack-scheduler
      - arc-runners-h100
      - zombie-cleanup
      - harbor-cache-recovery
      - monitoring
      - logging

  meta-prod-aws-ue1:
    cluster_name: meta-prod-aws-ue1
    region: us-east-1
    state_bucket: ciforge-tfstate-arc-cbr-prod-ue1
    access_config:
      authentication_mode: API_AND_CONFIG_MAP
      cluster_admin_role_names:
        - osdc_gha_prod
    base:
      vpc_cidr: "10.4.0.0/16"
    node_compactor:
      min_node_age_seconds: 300
    buildkit:
      amd64_instance_type: m6id.24xlarge
      amd64_pods_per_node: 2
      arm64_instance_type: m7gd.16xlarge
      arm64_pods_per_node: 4
      autoscaling:
        enabled: true
        amd64_min: 2   # 1x m6id.24xlarge (2 pods/node)
        amd64_max: 360  # ~90d peak ≈180, x2 for headroom
        arm64_min: 4   # 1x m7gd.16xlarge (4 pods/node)
        arm64_max: 30  # ~90d peak ≈15, x2 for headroom
        amd64_fallback: 32  # if KEDA can't read metrics, hold the proven fixed pool
        arm64_fallback: 8
    arc-runners:
      github_config_url: "https://github.com/pytorch"
      github_secret_name: meta-prod-aws-ue1
      runner_name_prefix: "mt-"
      runner_group: "meta-prod-aws-ue1"
      capacity_aware_fresh_multiplier: 0.8
      scheduler_name: bin-pack-scheduler
    modules:
      - karpenter
      - arc
      - hf-cache              # before nodepools: mount must be up before the node-init gate taint
      - nodepools
      - bin-pack-scheduler
      - arc-runners
      - keda
      - buildkit
      - zombie-cleanup
      - harbor-cache-recovery
      - monitoring
      - logging

  meta-prod-aws-ue2:
    cluster_name: meta-prod-aws-ue2
    region: us-east-2
    # state_bucket kept on the legacy arc-cbr name to avoid an S3 migration
    # (mirrors meta-prod-aws-ue1, which also kept its arc-cbr bucket).
    state_bucket: ciforge-tfstate-arc-cbr-prod
    access_config:
      authentication_mode: API_AND_CONFIG_MAP
      cluster_admin_role_names:
        - osdc_gha_prod
    base:
      vpc_cidr: "10.4.0.0/16"
    node_compactor:
      min_node_age_seconds: 300
    buildkit:
      amd64_instance_type: m6id.24xlarge
      amd64_pods_per_node: 2
      arm64_instance_type: m7gd.16xlarge
      arm64_pods_per_node: 4
      autoscaling:
        enabled: true
        amd64_min: 2   # 1x m6id.24xlarge (2 pods/node)
        amd64_max: 360  # ~90d peak ≈180, x2 for headroom
        arm64_min: 4   # 1x m7gd.16xlarge (4 pods/node)
        arm64_max: 30  # ~90d peak ≈15, x2 for headroom
        amd64_fallback: 32  # if KEDA can't read metrics, hold the proven fixed pool
        arm64_fallback: 8
    arc-runners:
      github_config_url: "https://github.com/pytorch"
      github_secret_name: meta-prod-aws-ue2
      runner_name_prefix: "mt-"
      runner_group: "meta-prod-aws-ue2"   # moved off the shared "default"/"release-runners" groups
      capacity_aware_fresh_multiplier: 0.8
      scheduler_name: bin-pack-scheduler
    arc-runners-h100:
      capacity_aware_fresh_multiplier: 1.0
    arc-runners-b200:
      capacity_aware_fresh_multiplier: 1.0
    nodepools-h100:
      # us-east-2 H100 capacity reservation (p5.48xlarge, 8× H100).
      capacity_reservation_ids:
        - cr-0c3f05dffb85ed832
    nodepools-b200:
      # us-east-2 B200 capacity reservation (p6-b200.48xlarge, 8× B200 each).
      # One reserved node total = 8 B200 GPUs. arc-runners-b200/defs/*.yaml
      # max_runners are sized assuming this count.
      capacity_reservation_ids:
        - cr-0b15c0b3163f09d26
    pypi_cache:
      replicas: 10
    modules:
      - karpenter
      - arc
      - hf-cache              # before nodepools: mount must be up before the node-init gate taint
      - nodepools
      - nodepools-b200
      - nodepools-h100
      - bin-pack-scheduler
      - arc-runners
      - arc-runners-b200
      - arc-runners-h100
      - keda
      - buildkit
      - zombie-cleanup
      - harbor-cache-recovery
      - monitoring
      - logging

  lf-prod-aws-ue1:
    region: us-east-1
    cluster_name: lf-prod-aws-ue1
    state_bucket: lf-osdc-tfstate-prod-ue1
    access_config:
      authentication_mode: API_AND_CONFIG_MAP
      cluster_admin_role_names:
        - lf_osdc_admin
    base:
      base_node_count: 4
      base_node_instance_type: m7i.4xlarge
      vpc_cidr: "10.12.0.0/16"
    node_compactor:
      min_node_age_seconds: 300
    arc-runners:
      github_config_url: "https://github.com/pytorch"
      github_secret_name: lf-prod-aws-ue1
      runner_name_prefix: "lf-"
      runner_group: lf-prod-aws-ue1
      scheduler_name: bin-pack-scheduler
      capacity_aware_fresh_multiplier: 0.8
    modules:
      - karpenter
      - arc
      - nodepools
      - bin-pack-scheduler
      - arc-runners
      - zombie-cleanup
      - harbor-cache-recovery
      - monitoring
      - logging

  lf-prod-aws-ue2:
    region: us-east-2
    cluster_name: lf-prod-aws-ue2
    state_bucket: lf-osdc-tfstate-prod-ue2
    access_config:
      authentication_mode: API_AND_CONFIG_MAP
      cluster_admin_role_names:
        - lf_osdc_admin
    base:
      base_node_count: 4
      base_node_instance_type: m7i.4xlarge
      vpc_cidr: "10.16.0.0/16"
    node_compactor:
      min_node_age_seconds: 300
    arc-runners:
      github_config_url: "https://github.com/pytorch"
      github_secret_name: lf-prod-aws-ue2
      runner_name_prefix: "lf-"
      runner_group: lf-prod-aws-ue2
      scheduler_name: bin-pack-scheduler
      capacity_aware_fresh_multiplier: 0.8
    modules:
      - karpenter
      - arc
      - nodepools
      - bin-pack-scheduler
      - arc-runners
      - zombie-cleanup
      - harbor-cache-recovery
      - monitoring
      - logging
