# justfile - Orchestrator for OSDC (Open Source Dev Cloud)
# https://just.systems/
#
# Reads clusters.yaml to determine what to deploy where.
# Each module under modules/ is independent and deployed in order.
#
# Working directory: osdc/
#
# IMPORTANT: Every recipe that interacts with a live cluster (kubectl, helm,
# deploy scripts, tests) MUST call `just kubeconfig <cluster>` or the
# equivalent `aws eks update-kubeconfig` before doing any k8s work. This
# prevents user error from operating on the wrong cluster. When adding new
# recipes that touch a cluster, always include the kubeconfig step.
#
# Usage:
#   just list                          # show clusters and their modules
#   just deploy meta-staging-aws-uw1            # full deploy (base + all modules)
#   just deploy-base meta-staging-aws-uw1       # base infra only
#   just deploy-module meta-staging-aws-uw1 arc # single module
#   just test                            # run all tests
#   just lint                            # lint all code

set dotenv-load := true
set shell := ["mise", "exec", "--", "bash", "-euo", "pipefail", "-c"]

ROOT := justfile_directory()
UPSTREAM := env_var_or_default("OSDC_UPSTREAM", ROOT)
CLUSTERS_YAML := ROOT / "clusters.yaml"
SCRIPTS := UPSTREAM / "scripts"
CFG := SCRIPTS / "cluster-config.py"

# ============================================================================
# SETUP
# ============================================================================

# Install Python dependencies
setup:
    @echo "Installing Python dependencies..."
    @cd {{UPSTREAM}} && uv sync
    @echo "Setup complete."

# Remove all caches, generated files, and initialization artifacts
clean:
    #!/usr/bin/env bash
    set -euo pipefail
    echo "Cleaning caches and initialization artifacts..."
    echo ""

    REMOVED=0

    _rm() {
        for target in "$@"; do
            [[ -e "$target" ]] || continue
            echo "  rm -rf $target"
            rm -rf "$target"
            REMOVED=$((REMOVED + 1))
        done
    }

    ROOTS=("{{ROOT}}")
    [[ "{{ROOT}}" != "{{UPSTREAM}}" ]] && ROOTS+=("{{UPSTREAM}}")

    for root in "${ROOTS[@]}"; do
        echo "── ${root} ──"

        # Tofu init caches
        while IFS= read -r -d '' d; do _rm "$d"; done \
            < <(find "$root" -path '*/.scratch' -prune -o -path '*/.venv' -prune -o \
                -type d -name '.terraform' -print0 2>/dev/null)

        # Tofu lock files and plan files
        while IFS= read -r -d '' f; do _rm "$f"; done \
            < <(find "$root" -path '*/.scratch' -prune -o -path '*/.venv' -prune -o \
                \( -name '.terraform.lock.hcl' -o -name 'tfplan' -o -name '*.tfplan' \) -type f -print0 2>/dev/null)

        # Python bytecode and tool caches
        while IFS= read -r -d '' d; do _rm "$d"; done \
            < <(find "$root" -path '*/.scratch' -prune -o -path '*/.venv' -prune -o \
                -type d \( -name '__pycache__' -o -name '.pytest_cache' -o -name '.ruff_cache' \) -print0 2>/dev/null)

        # Coverage artifacts
        _rm "$root/.coverage" "$root/coverage.json"

        # Generated module outputs
        if [[ -d "$root/modules" ]]; then
            while IFS= read -r -d '' d; do _rm "$d"; done \
                < <(find "$root/modules" -type d -name 'generated' -print0 2>/dev/null)
        fi

        # Python virtual environment
        _rm "$root/.venv"

        # Integration test scratch
        _rm "$root/.scratch"
    done

    echo ""
    if [[ $REMOVED -eq 0 ]]; then
        echo "Nothing to clean."
    else
        echo "Removed $REMOVED item(s). Run 'just setup' to re-initialize."
    fi

# ============================================================================
# INFO
# ============================================================================

# List all clusters and their modules
list:
    @echo "Clusters defined in clusters.yaml:"
    @echo ""
    @export CLUSTERS_YAML="{{CLUSTERS_YAML}}"; \
    for cid in $(uv run {{CFG}} --list); do \
        region=$(uv run {{CFG}} "$cid" region); \
        cname=$(uv run {{CFG}} "$cid" cluster_name); \
        modules=$(uv run {{CFG}} "$cid" modules | tr '\n' ' '); \
        echo "  $cid ($cname @ $region)"; \
        echo "    modules: $modules"; \
        echo ""; \
    done

# Show what would be deployed for a cluster (dry run)
show cluster:
    @export CLUSTERS_YAML="{{CLUSTERS_YAML}}"; \
    echo "Cluster: {{cluster}}"; \
    echo "  Name:    $(uv run {{CFG}} {{cluster}} cluster_name)"; \
    echo "  Region:  $(uv run {{CFG}} {{cluster}} region)"; \
    echo "  Bucket:  $(uv run {{CFG}} {{cluster}} state_bucket)"; \
    echo "  Modules:"; \
    uv run {{CFG}} {{cluster}} modules | sed 's/^/    - /'; \
    echo ""; \
    echo "  Tofu vars:"; \
    uv run {{CFG}} {{cluster}} tfvars | tr ' ' '\n' | sed 's/^/    /'

# Show deploy audit log ConfigMaps for a cluster
deploy-history cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"

    just kubeconfig "$CLUSTER"

    echo "Deploy history for cluster: $CLUSTER"
    echo ""
    kubectl get configmaps -n osdc-system \
        -l app.kubernetes.io/managed-by=osdc-deploy-log \
        --sort-by=.metadata.creationTimestamp \
        -o custom-columns='NAME:.metadata.name,AGE:.metadata.creationTimestamp'

# Human-readable deploy status: current versions and recent history
deploy-status cluster name='':
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    NAME="{{name}}"

    just kubeconfig "$CLUSTER"

    ARGS=("$CLUSTER")
    [[ -n "$NAME" ]] && ARGS+=("$NAME")

    kubectl get configmaps -n osdc-system \
        -l app.kubernetes.io/managed-by=osdc-deploy-log \
        -o json \
    | python3 "{{SCRIPTS}}/deploy-status.py" "${ARGS[@]}"

# Update kubeconfig for a cluster (aws eks update-kubeconfig)
kubeconfig cluster:
    @export CLUSTERS_YAML="{{CLUSTERS_YAML}}"; \
    CNAME=$(uv run {{CFG}} {{cluster}} cluster_name); \
    REGION=$(uv run {{CFG}} {{cluster}} region); \
    echo "Updating kubeconfig for $CNAME ($REGION)..."; \
    NO_PROXY="${NO_PROXY:-},.eks.amazonaws.com" no_proxy="${no_proxy:-},.eks.amazonaws.com" \
    "{{UPSTREAM}}/scripts/kubeconfig-lock.sh" --name "$CNAME" --region "$REGION" --alias "$CNAME"

# ============================================================================
# BOOTSTRAP
# ============================================================================

# Bootstrap S3 state bucket + DynamoDB lock table for a cluster
bootstrap cluster:
    @OSDC_ROOT="{{ROOT}}" OSDC_UPSTREAM="{{UPSTREAM}}" CLUSTERS_YAML="{{CLUSTERS_YAML}}" {{SCRIPTS}}/bootstrap-state.sh {{cluster}}

# Bootstrap all clusters
bootstrap-all:
    @OSDC_ROOT="{{ROOT}}" OSDC_UPSTREAM="{{UPSTREAM}}" CLUSTERS_YAML="{{CLUSTERS_YAML}}" {{SCRIPTS}}/bootstrap-state.sh --all

# ============================================================================
# DEPLOY
# ============================================================================

# Full deploy: base infra + all modules in order
deploy cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)
    SECONDS=0

    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo "FULL DEPLOYMENT: $CLUSTER ($CNAME)"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo ""
    echo "Modules:"
    uv run {{CFG}} "$CLUSTER" modules | sed 's/^/  - /'
    echo ""
    CONFIRM="${OSDC_CONFIRM:-ask}"
    if [ "$CONFIRM" = "yes" ]; then
        echo "Auto-confirmed (OSDC_CONFIRM=yes)"
    elif [ "$CONFIRM" = "no" ]; then
        echo "Cancelled (OSDC_CONFIRM=no)."
        exit 1
    else
        read -p "Continue? [y/N] " -n 1 -r
        echo
        [[ $REPLY =~ ^[Yy]$ ]] || { echo "Cancelled."; exit 1; }
    fi

    # Deploy audit logging (after confirmation to avoid orphaned "started" entries)
    source "{{UPSTREAM}}/scripts/deploy-log.sh"
    DEPLOY_LOG_START=$(deploy_log_start cmd "$CLUSTER" "deploy")
    trap 'deploy_log_finish cmd "$CLUSTER" "deploy" "$DEPLOY_LOG_START" failed' ERR

    just deploy-base "$CLUSTER"

    for module in $(uv run {{CFG}} "$CLUSTER" modules); do
        just deploy-module "$CLUSTER" "$module"
    done

    # Recycle Karpenter nodes if configured (e.g., staging — ensures fresh userData/AMI)
    RECYCLE=$(uv run {{CFG}} "$CLUSTER" recycle_karpenter_nodes false)
    if [ "$RECYCLE" = "true" ]; then
        echo ""
        echo "── Recycling Karpenter nodes (recycle_karpenter_nodes=true) ──"
        just recycle-nodes "$CLUSTER"
    fi

    echo ""
    if (( SECONDS < 60 )); then elapsed="${SECONDS}s"; else elapsed="$((SECONDS / 60))m$((SECONDS % 60))s"; fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo "DEPLOYMENT COMPLETE: $CLUSTER ($elapsed)"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    # Deploy audit logging — record successful completion
    trap - ERR
    deploy_log_finish cmd "$CLUSTER" "deploy" "$DEPLOY_LOG_START"

    # Taint ARC runner nodes for graceful refresh (production only)
    # When nodes are recycled (staging), tainting is pointless — they're already being destroyed.
    # OSDC_TAINT_NODES=yes|no|ask (default: ask)
    if [ "$RECYCLE" != "true" ]; then
        TAINT="${OSDC_TAINT_NODES:-ask}"
        if [ "$TAINT" = "yes" ]; then
            echo ""
            echo "── Tainting ARC runner nodes (OSDC_TAINT_NODES=yes) ──"
            just taint-nodes "$CLUSTER"
        elif [ "$TAINT" = "no" ]; then
            echo ""
            echo "Skipping node tainting (OSDC_TAINT_NODES=no)"
        else
            echo ""
            echo "Taint existing ARC runner nodes with NoSchedule?"
            echo "  This prevents new jobs from scheduling on old nodes while"
            echo "  in-flight jobs finish undisturbed. Fresh nodes will be"
            echo "  provisioned by Karpenter as needed."
            read -p "Taint runner nodes? [y/N] " -n 1 -r
            echo
            if [[ $REPLY =~ ^[Yy]$ ]]; then
                just taint-nodes "$CLUSTER"
            else
                echo "Skipping node tainting."
            fi
        fi
    fi

    # Run smoke tests: OSDC_SMOKE=yes|no|ask (default: ask)
    RUN_SMOKE="${OSDC_SMOKE:-ask}"
    if [ "$RUN_SMOKE" = "yes" ]; then
        echo ""
        echo "── Running smoke tests (OSDC_SMOKE=yes) ──"
        just smoke "$CLUSTER"
    elif [ "$RUN_SMOKE" = "no" ]; then
        echo ""
        echo "Skipping smoke tests (OSDC_SMOKE=no)"
    else
        echo ""
        read -p "Run smoke tests? [y/N] " -n 1 -r
        echo
        if [[ $REPLY =~ ^[Yy]$ ]]; then
            just smoke "$CLUSTER"
        else
            echo "Skipping smoke tests."
        fi
    fi

# Deploy base infrastructure (tofu + harbor + base k8s)
deploy-base cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    source "{{UPSTREAM}}/scripts/state-config.sh"
    : "${STATE_REGION:?state-config.sh did not export STATE_REGION}"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    REGION=$(uv run {{CFG}} "$CLUSTER" region)
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)
    BUCKET=$(uv run {{CFG}} "$CLUSTER" state_bucket)
    TFVARS=$(uv run {{CFG}} "$CLUSTER" tfvars)

    # Preflight: check state bucket exists
    if ! aws s3api head-bucket --bucket "${BUCKET}" --region "${STATE_REGION}" 2>/dev/null; then
        echo ""
        echo "ERROR: State bucket '${BUCKET}' does not exist."
        echo ""
        echo "Run bootstrap first:"
        echo "  just bootstrap ${CLUSTER}"
        echo ""
        exit 1
    fi

    # Deploy audit logging (after preflight to avoid orphaned "started" entries)
    source "{{UPSTREAM}}/scripts/deploy-log.sh"
    DEPLOY_LOG_START=$(deploy_log_start cmd "$CLUSTER" "deploy-base")
    trap 'deploy_log_finish cmd "$CLUSTER" "deploy-base" "$DEPLOY_LOG_START" failed' ERR

    echo ""
    echo "━━━ modules/eks/terraform ━━━"
    cd {{UPSTREAM}}/modules/eks/terraform
    tofu init -reconfigure \
        -backend-config="bucket=${BUCKET}" \
        -backend-config="key=${CLUSTER}/base/terraform.tfstate" \
        -backend-config="region=${STATE_REGION}" \
        -backend-config="dynamodb_table=ciforge-terraform-locks"

    # Suspend ERR trap — tofu plan returns exit code 2 for "changes detected"
    trap - ERR
    set +e
    eval tofu plan -lock-timeout=15m $TFVARS -out=tfplan -detailed-exitcode
    PLAN_EXIT=$?
    set -e
    trap 'deploy_log_finish cmd "$CLUSTER" "deploy-base" "$DEPLOY_LOG_START" failed' ERR

    if [[ $PLAN_EXIT -eq 0 ]]; then
        echo "No changes. Skipping apply."
        rm -f tfplan
    elif [[ $PLAN_EXIT -eq 2 ]]; then
        echo ""
        CONFIRM="${OSDC_CONFIRM:-ask}"
        if [[ "$CONFIRM" == "yes" ]]; then
            echo "Auto-confirmed (OSDC_CONFIRM=yes)"
        elif [[ "$CONFIRM" == "no" ]]; then
            rm -f tfplan
            echo "Cancelled (OSDC_CONFIRM=no)."
            exit 1
        else
            read -p "Apply this plan? [y/N] " -n 1 -r
            echo
            if [[ ! $REPLY =~ ^[Yy]$ ]]; then
                rm -f tfplan
                echo "Cancelled."
                exit 1
            fi
        fi
        tofu apply -lock-timeout=15m tfplan
        rm -f tfplan
    else
        rm -f tfplan
        echo "Tofu plan failed."
        deploy_log_finish cmd "$CLUSTER" "deploy-base" "$DEPLOY_LOG_START" failed
        exit 1
    fi

    echo ""
    echo "Updating kubeconfig for $CNAME ($REGION)..."
    NO_PROXY="${NO_PROXY:-},.eks.amazonaws.com" no_proxy="${no_proxy:-},.eks.amazonaws.com" \
      "{{UPSTREAM}}/scripts/kubeconfig-lock.sh" --name "$CNAME" --region "$REGION" --alias "$CNAME"

    echo ""
    echo "━━━ BASE: Mirror bootstrap images ━━━"
    {{UPSTREAM}}/modules/eks/scripts/mirror-images.sh "$CLUSTER"

    echo ""
    echo "━━━ BASE: Kubernetes resources ━━━"
    kubectl apply -k {{UPSTREAM}}/base/kubernetes/

    echo ""
    echo "━━━ BASE: Deploy node-taint-remover shared library ━━━"
    {{UPSTREAM}}/base/kubernetes/node-taint-remover/deploy.sh "$CLUSTER"

    echo ""
    echo "━━━ BASE: Deploy ENIConfigs ━━━"
    {{UPSTREAM}}/base/kubernetes/eniconfigs/deploy.sh "$CLUSTER"

    echo ""
    echo "━━━ BASE: Deploy Harbor ━━━"
    cd "{{ROOT}}"
    just _deploy-harbor "$CLUSTER"

    echo ""
    echo "━━━ BASE: Deploy Node Compactor ━━━"
    {{UPSTREAM}}/base/node-compactor/deploy.sh "$CLUSTER"

    echo ""
    echo "━━━ BASE: Deploy Image Cache Janitor ━━━"
    {{UPSTREAM}}/base/kubernetes/image-cache-janitor/deploy.sh "$CLUSTER"

    echo ""
    echo "━━━ BASE: Deploy NodeLocal DNSCache ━━━"
    {{UPSTREAM}}/base/kubernetes/nodelocaldns/deploy.sh "$CLUSTER"

    echo ""
    echo "Base deployment complete."

    # Deploy audit logging — record successful completion
    trap - ERR
    deploy_log_finish cmd "$CLUSTER" "deploy-base" "$DEPLOY_LOG_START"

# Deploy a single module to a cluster
deploy-module cluster module force="":
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    source "{{UPSTREAM}}/scripts/state-config.sh"
    : "${STATE_REGION:?state-config.sh did not export STATE_REGION}"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    if [[ -n "{{force}}" ]]; then
        export HELM_FORCE_UPGRADE=1
    fi
    CLUSTER="{{cluster}}"
    MODULE="{{module}}"
    REGION=$(uv run {{CFG}} "$CLUSTER" region)
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)

    # Ensure kubectl is configured for the target cluster
    just kubeconfig "$CLUSTER"

    echo ""
    echo "━━━ MODULE: $MODULE ━━━"

    # Check module is enabled for this cluster
    if ! uv run {{CFG}} "$CLUSTER" has-module "$MODULE"; then
        echo "Module '$MODULE' is not enabled for cluster '$CLUSTER'. Skipping."
        exit 0
    fi

    # Module resolution: local modules override upstream
    if [[ -d "{{ROOT}}/modules/$MODULE" ]]; then
        MODULE_DIR="{{ROOT}}/modules/$MODULE"
    elif [[ -d "{{UPSTREAM}}/modules/$MODULE" ]]; then
        MODULE_DIR="{{UPSTREAM}}/modules/$MODULE"
    else
        echo "Error: module '$MODULE' not found in modules/ or upstream."
        exit 1
    fi

    # Deploy audit logging — record start (command-level + module-level)
    source "{{UPSTREAM}}/scripts/deploy-log.sh"
    DEPLOY_LOG_CMD_START=$(deploy_log_start cmd "$CLUSTER" "deploy-module-$MODULE")
    DEPLOY_LOG_MOD_START=$(deploy_log_start module "$CLUSTER" "$MODULE")
    trap 'deploy_log_finish cmd "$CLUSTER" "deploy-module-$MODULE" "$DEPLOY_LOG_CMD_START" failed; deploy_log_finish module "$CLUSTER" "$MODULE" "$DEPLOY_LOG_MOD_START" failed' ERR

    # Phase 1: Terraform (if module has its own)
    if [[ -f "$MODULE_DIR/terraform/main.tf" ]]; then
        echo "  Applying terraform for $MODULE..."
        BUCKET=$(uv run {{CFG}} "$CLUSTER" state_bucket)
        cd "$MODULE_DIR/terraform"
        tofu init -reconfigure \
            -backend-config="bucket=${BUCKET}" \
            -backend-config="key=${CLUSTER}/${MODULE}/terraform.tfstate" \
            -backend-config="region=${STATE_REGION}" \
            -backend-config="dynamodb_table=ciforge-terraform-locks"

        # Modules get cluster_name and aws_region as minimum vars
        # Suspend ERR trap — tofu plan returns exit code 2 for "changes detected"
        trap - ERR
        set +e
        tofu plan -lock-timeout=15m \
            -var="cluster_name=${CNAME}" \
            -var="aws_region=${REGION}" \
            -var="state_bucket=${BUCKET}" \
            -var="cluster_id=${CLUSTER}" \
            -out=tfplan -detailed-exitcode
        PLAN_EXIT=$?
        set -e
        trap 'deploy_log_finish cmd "$CLUSTER" "deploy-module-$MODULE" "$DEPLOY_LOG_CMD_START" failed; deploy_log_finish module "$CLUSTER" "$MODULE" "$DEPLOY_LOG_MOD_START" failed' ERR

        if [[ $PLAN_EXIT -eq 0 ]]; then
            echo "  No changes. Skipping apply."
            rm -f tfplan
        elif [[ $PLAN_EXIT -eq 2 ]]; then
            tofu apply -lock-timeout=15m tfplan
            rm -f tfplan
        else
            rm -f tfplan
            echo "  Tofu plan failed."
            deploy_log_finish cmd "$CLUSTER" "deploy-module-$MODULE" "$DEPLOY_LOG_CMD_START" failed
            deploy_log_finish module "$CLUSTER" "$MODULE" "$DEPLOY_LOG_MOD_START" failed
            exit 1
        fi
        cd -
    fi

    # Phase 2: Kubernetes resources (if module has them)
    if [[ -f "$MODULE_DIR/kubernetes/kustomization.yaml" ]]; then
        echo "  Applying kubernetes resources for $MODULE..."
        kubectl apply -k "$MODULE_DIR/kubernetes/"
    fi

    # Phase 3: Module-specific deploy script (if exists)
    if [[ -x "$MODULE_DIR/deploy.sh" ]]; then
        echo "  Running $MODULE deploy script..."
        "$MODULE_DIR/deploy.sh" "$CLUSTER" "$CNAME" "$REGION"
    fi

    echo "  Module $MODULE deployed."

    # Deploy audit logging — record successful completion
    trap - ERR
    deploy_log_finish module "$CLUSTER" "$MODULE" "$DEPLOY_LOG_MOD_START"
    deploy_log_finish cmd "$CLUSTER" "deploy-module-$MODULE" "$DEPLOY_LOG_CMD_START"

# Operator is expected to have run `just drain-runners` first for runner modules.
# Sweeps every namespaced and cluster-scoped kind labelled osdc.io/module=<module>,
# helm-uninstalls matching arc-runners releases, drops the module namespace (if
# labelled), and runs `tofu destroy` on the module's per-cluster terraform root
# (if any). Honors OSDC_CONFIRM=yes|no|ask (default ask).
# Remove a module's resources (kubernetes + per-cluster terraform) from a cluster.
remove-module cluster module:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    source "{{UPSTREAM}}/scripts/state-config.sh"
    : "${STATE_REGION:?state-config.sh did not export STATE_REGION}"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    MODULE="{{module}}"

    if [[ -z "$CLUSTER" ]] || [[ -z "$MODULE" ]]; then
        echo "Usage: just remove-module <cluster> <module>" >&2
        exit 2
    fi

    just kubeconfig "$CLUSTER"

    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)
    REGION=$(uv run {{CFG}} "$CLUSTER" region)
    BUCKET=$(uv run {{CFG}} "$CLUSTER" state_bucket)

    # Module resolution mirrors deploy-module: consumer first, then upstream.
    MODULE_DIR=""
    if [[ -d "$OSDC_ROOT/modules/$MODULE" ]]; then
        MODULE_DIR="$OSDC_ROOT/modules/$MODULE"
    elif [[ -d "$OSDC_UPSTREAM/modules/$MODULE" ]]; then
        MODULE_DIR="$OSDC_UPSTREAM/modules/$MODULE"
    fi
    HAS_TF="no"
    if [[ -n "$MODULE_DIR" ]] && [[ -f "$MODULE_DIR/terraform/main.tf" ]]; then
        HAS_TF="yes"
    fi

    NS_KINDS="deployment,daemonset,statefulset,service,configmap,secret,serviceaccount,role,rolebinding,networkpolicy,pdb,pvc"
    CLUSTER_KINDS=(
        clusterrole
        clusterrolebinding
        storageclass
        nodepool.karpenter.sh
        ec2nodeclass.karpenter.k8s.aws
    )

    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo "REMOVE MODULE: $MODULE from $CLUSTER ($CNAME)"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo ""
    echo "Selector: osdc.io/module=$MODULE"
    echo ""

    echo "── Planned actions ──"
    echo "  1. helm uninstall arc-runner-hook-* matching releases (arc-runners convention)"
    echo "  2. kubectl delete -A on namespaced kinds: $NS_KINDS"
    echo "  3. kubectl delete on cluster-scoped kinds: ${CLUSTER_KINDS[*]}"
    echo "  4. kubectl delete namespace (only if labelled osdc.io/module=$MODULE)"
    if [[ "$HAS_TF" == "yes" ]]; then
        echo "  5. tofu destroy: $MODULE_DIR/terraform (state key: $CLUSTER/$MODULE/terraform.tfstate)"
    else
        echo "  5. tofu destroy: (skipped — no $MODULE/terraform/main.tf)"
    fi
    echo ""

    echo "── Resources currently matching osdc.io/module=$MODULE ──"
    echo "  Namespaced ($NS_KINDS):"
    kubectl get "$NS_KINDS" -A -l "osdc.io/module=$MODULE" --ignore-not-found 2>/dev/null \
        | sed 's/^/    /' || echo "    (query failed or none found)"
    echo ""
    echo "  Cluster-scoped:"
    for k in "${CLUSTER_KINDS[@]}"; do
        OUT=$(kubectl get "$k" -l "osdc.io/module=$MODULE" --ignore-not-found 2>/dev/null || true)
        if [[ -n "$OUT" ]]; then
            echo "    [$k]"
            echo "$OUT" | sed 's/^/      /'
        fi
    done
    echo ""
    echo "  Namespaces:"
    kubectl get namespace -l "osdc.io/module=$MODULE" --ignore-not-found 2>/dev/null \
        | sed 's/^/    /' || echo "    (none)"
    echo ""

    CONFIRM="${OSDC_CONFIRM:-ask}"
    if [[ "$CONFIRM" == "yes" ]]; then
        echo "Auto-confirmed (OSDC_CONFIRM=yes)"
    elif [[ "$CONFIRM" == "no" ]]; then
        echo "Cancelled (OSDC_CONFIRM=no)."
        exit 1
    else
        read -p "Proceed with removal? [y/N] " -n 1 -r
        echo
        [[ $REPLY =~ ^[Yy]$ ]] || { echo "Cancelled."; exit 1; }
    fi
    echo ""

    PARTIAL=0

    # ── Phase 1: arc-runner-hook helm uninstall ────────────────────────────
    # arc-runners convention: arc-runner-hook-<n> ConfigMap → arc-<n> Helm release.
    # Must run BEFORE the namespaced-kind sweep deletes those ConfigMaps,
    # otherwise we lose the mapping to discover release names.
    echo "── Phase 1: helm uninstall arc-runner-hook releases ──"
    HOOK_CMS=$(kubectl get cm -A -l "osdc.io/module=$MODULE" \
        -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' \
        2>/dev/null || true)
    if [[ -z "$HOOK_CMS" ]]; then
        echo "  No arc-runner-hook ConfigMaps found for module $MODULE."
    else
        FOUND=0
        while IFS=/ read -r ns name; do
            [ -z "$name" ] && continue
            [[ "$name" == arc-runner-hook-* ]] || continue
            release="arc-${name#arc-runner-hook-}"
            FOUND=$((FOUND + 1))
            echo "  helm uninstall $release -n $ns"
            helm uninstall "$release" -n "$ns" --ignore-not-found || PARTIAL=1
            kubectl delete secret -n "$ns" -l "owner=helm,name=$release" --ignore-not-found || PARTIAL=1
        done <<< "$HOOK_CMS"
        [[ $FOUND -eq 0 ]] && echo "  No arc-runner-hook-* ConfigMaps among labelled CMs."
    fi
    echo ""

    # ── Phase 2: namespaced kinds ──────────────────────────────────────────
    echo "── Phase 2: delete namespaced kinds labelled osdc.io/module=$MODULE ──"
    if ! kubectl delete "$NS_KINDS" -A -l "osdc.io/module=$MODULE" --ignore-not-found; then
        PARTIAL=1
    fi
    echo ""

    # ── Phase 3: cluster-scoped kinds ──────────────────────────────────────
    echo "── Phase 3: delete cluster-scoped kinds labelled osdc.io/module=$MODULE ──"
    for k in "${CLUSTER_KINDS[@]}"; do
        if ! kubectl delete "$k" -l "osdc.io/module=$MODULE" --ignore-not-found; then
            PARTIAL=1
        fi
    done
    echo ""

    # ── Phase 4: namespace (cascade defense in depth) ──────────────────────
    # Only delete namespaces that carry the module label — never fabricate or
    # guess names. Anything left in the namespace that lacked the per-resource
    # label dies with the namespace.
    echo "── Phase 4: delete namespace(s) labelled osdc.io/module=$MODULE ──"
    if ! kubectl delete namespace -l "osdc.io/module=$MODULE" --ignore-not-found; then
        PARTIAL=1
    fi
    echo ""

    # ── Phase 5: tofu destroy (per-cluster terraform root, if any) ─────────
    echo "── Phase 5: tofu destroy per-cluster terraform state ──"
    if [[ "$HAS_TF" == "yes" ]]; then
        echo "  Destroying $MODULE_DIR/terraform (state key: $CLUSTER/$MODULE/terraform.tfstate)"
        cd "$MODULE_DIR/terraform"
        if ! tofu init -reconfigure \
            -backend-config="bucket=${BUCKET}" \
            -backend-config="key=${CLUSTER}/${MODULE}/terraform.tfstate" \
            -backend-config="region=${STATE_REGION}" \
            -backend-config="dynamodb_table=ciforge-terraform-locks"; then
            echo "  ERROR: tofu init failed for $MODULE." >&2
            PARTIAL=1
        else
            # OSDC_CONFIRM=yes → -auto-approve; otherwise let tofu prompt.
            DESTROY_ARGS=(
                -lock-timeout=15m
                -var="cluster_name=${CNAME}"
                -var="aws_region=${REGION}"
                -var="state_bucket=${BUCKET}"
                -var="cluster_id=${CLUSTER}"
            )
            [[ "$CONFIRM" == "yes" ]] && DESTROY_ARGS+=(-auto-approve)
            if ! tofu destroy "${DESTROY_ARGS[@]}"; then
                echo "  ERROR: tofu destroy failed for $MODULE." >&2
                PARTIAL=1
            fi
        fi
        cd - >/dev/null
    else
        echo "  Skipped — module has no per-cluster terraform root."
        echo "  (Sub-roots like $MODULE/terraform/<subdir>/main.tf are NOT touched by remove-module.)"
    fi
    echo ""

    # ── Operator reminders (module-specific) ───────────────────────────────
    if [[ "$MODULE" == "cache-enforcer" ]]; then
        echo "━━━ OPERATOR WARNING: cache-enforcer requires nodepools redeploy ━━━"
        echo "modules/nodepools/scripts/python/generate_nodepools.py gates the"
        echo "node-init.osdc.io/cache-enforcer=true startup taint on the presence"
        echo "of 'cache-enforcer' in the cluster's modules list. After dropping"
        echo "cache-enforcer from clusters.yaml, the rendered NodePool spec on"
        echo "the cluster still emits that taint until nodepools is redeployed."
        echo "Fresh Karpenter nodes will be tainted with nothing to clear them"
        echo "(the DaemonSet that removed the taint is now gone) and will never"
        echo "schedule pods. Redeploy nodepools BEFORE recycling nodes:"
        echo "  just deploy-module $CLUSTER nodepools"
        echo ""
        echo "━━━ OPERATOR WARNING: cache-enforcer iptables persistence ━━━"
        echo "Removing the DaemonSet does NOT undo iptables rules already installed"
        echo "on runner nodes. Every node that ever ran cache-enforcer still has the"
        echo "CACHE_ENFORCER chain wired into OUTPUT (both iptables and ip6tables)."
        echo ""
        echo "Recommended: recycle the affected nodes so fresh nodes come up clean"
        echo "(after the nodepools redeploy above):"
        echo "  just recycle-nodes $CLUSTER"
        echo ""
        echo "Verify a node is still affected (any non-empty output = chain present):"
        echo "  kubectl debug node/<node> -it --image=alpine -- chroot /host sh -c \\"
        echo "    'iptables -S CACHE_ENFORCER 2>/dev/null; ip6tables -S CACHE_ENFORCER 2>/dev/null'"
        echo ""
    fi
    if [[ "$MODULE" == "pypi-cache" ]]; then
        echo "━━━ OPERATOR WARNING: stale env vars on existing runner pods ━━━"
        echo "Workflow pods scheduled before this removal still carry PIP_INDEX_URL,"
        echo "UV_DEFAULT_INDEX, and the rest of the pypi-cache env block pointing at"
        echo "pypi-cache-cpu.pypi-cache.svc.cluster.local. The Service is gone, so"
        echo "DNS will NXDOMAIN and pip/uv installs in those pods will fail."
        echo "Drain in-flight runners and recycle nodes to roll fresh pods that get"
        echo "the regenerated (pypi-cache-less) ConfigMap:"
        echo "  just drain-runners $CLUSTER && just recycle-nodes $CLUSTER"
        echo ""
        echo "━━━ OPERATOR WARNING: EFS wheelhouse data dropped ━━━"
        echo "tofu destroy wiped the per-cluster aws_efs_file_system.pypi_cache,"
        echo "which held the cached .whl files. Recoverable: the shared S3 bucket"
        echo "s3://pytorch-pypi-wheel-cache/{slug}/ (account-wide, NOT destroyed)"
        echo "is the source of truth. On the next pypi-cache deploy, wheel-syncer"
        echo "re-pulls everything from S3 — expect slow first-cache-warm and"
        echo "elevated pip install latency on the first jobs after redeploy."
        echo ""
        echo "━━━ OPERATOR WARNING: pypi-cache leftovers in monitoring module ━━━"
        echo "The 'monitoring' module owns modules/monitoring/.../pypi-cache.yaml"
        echo "ServiceMonitor independently of the pypi-cache module. After this"
        echo "removal it will go stale (no matching endpoints) — harmless, but"
        echo "consider redeploying monitoring to drop the orphan once pypi-cache"
        echo "is permanently gone from this cluster."
        echo ""
    fi

    if [[ $PARTIAL -eq 0 ]]; then
        echo "Module $MODULE removed from $CLUSTER."
    else
        echo "Module $MODULE partially removed from $CLUSTER — see errors above."
        exit 1
    fi

# Run `tofu plan` for base + every terraform-backed module of a cluster.
# Read-only: no apply, no k8s/helm side effects. Safe to run from CI on PRs.
plan cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    source "{{UPSTREAM}}/scripts/state-config.sh"
    : "${STATE_REGION:?state-config.sh did not export STATE_REGION}"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    REGION=$(uv run {{CFG}} "$CLUSTER" region)
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)
    BUCKET=$(uv run {{CFG}} "$CLUSTER" state_bucket)
    TFVARS=$(uv run {{CFG}} "$CLUSTER" tfvars)

    if ! aws s3api head-bucket --bucket "${BUCKET}" --region "${STATE_REGION}" 2>/dev/null; then
        echo "ERROR: State bucket '${BUCKET}' does not exist. Run: just bootstrap ${CLUSTER}"
        exit 1
    fi

    echo "━━━ PLAN: Base (${CLUSTER}) ━━━"
    cd {{UPSTREAM}}/modules/eks/terraform
    tofu init -reconfigure -input=false -no-color \
        -backend-config="bucket=${BUCKET}" \
        -backend-config="key=${CLUSTER}/base/terraform.tfstate" \
        -backend-config="region=${STATE_REGION}" \
        -backend-config="dynamodb_table=ciforge-terraform-locks" >/dev/null
    eval tofu plan -lock-timeout=15m -input=false -no-color $TFVARS

    for MODULE in $(uv run {{CFG}} "$CLUSTER" modules); do
        if [[ -d "{{ROOT}}/modules/$MODULE" ]]; then
            MODULE_DIR="{{ROOT}}/modules/$MODULE"
        elif [[ -d "{{UPSTREAM}}/modules/$MODULE" ]]; then
            MODULE_DIR="{{UPSTREAM}}/modules/$MODULE"
        else
            continue
        fi
        [[ -f "$MODULE_DIR/terraform/main.tf" ]] || continue

        echo ""
        echo "━━━ PLAN: Module $MODULE (${CLUSTER}) ━━━"
        cd "$MODULE_DIR/terraform"
        tofu init -reconfigure -input=false -no-color \
            -backend-config="bucket=${BUCKET}" \
            -backend-config="key=${CLUSTER}/${MODULE}/terraform.tfstate" \
            -backend-config="region=${STATE_REGION}" \
            -backend-config="dynamodb_table=ciforge-terraform-locks" >/dev/null
        tofu plan -lock-timeout=15m -input=false -no-color \
            -var="cluster_name=${CNAME}" \
            -var="aws_region=${REGION}" \
            -var="state_bucket=${BUCKET}" \
            -var="cluster_id=${CLUSTER}"
    done

# Force-release a stuck OpenTofu state lock for a cluster's base or module state.
# slot = "base" (modules/eks/terraform) or a module name (modules/<slot>/terraform).
# lock-id is the UUID from tofu's "Lock Info" block (the ID: field), NOT the DynamoDB key.
# tofu prompts for confirmation before releasing the lock.
# DANGER: confirm no deploy is in progress first — force-unlocking a live apply corrupts state.
force-unlock cluster slot lock_id:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    source "{{UPSTREAM}}/scripts/state-config.sh"
    : "${STATE_REGION:?state-config.sh did not export STATE_REGION}"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    SLOT="{{slot}}"
    LOCK_ID="{{lock_id}}"

    if [[ -z "$CLUSTER" ]] || [[ -z "$SLOT" ]] || [[ -z "$LOCK_ID" ]]; then
        echo "Usage: just force-unlock <cluster> <base|module> <lock-id>" >&2
        exit 2
    fi

    # Base state lives under slot 'base' even though its code is in modules/eks/terraform.
    # Reject 'eks' so an operator doesn't clear a phantom <cluster>/eks lock and leave the
    # real <cluster>/base lock held.
    if [[ "$SLOT" == "eks" ]]; then
        echo "Error: base state uses slot 'base', not 'eks'. Run: just force-unlock $CLUSTER base $LOCK_ID" >&2
        exit 2
    fi

    # Resolve the terraform root for this state slot (consumer modules override upstream).
    if [[ "$SLOT" == "base" ]]; then
        TF_DIR="{{UPSTREAM}}/modules/eks/terraform"
    elif [[ -d "{{ROOT}}/modules/$SLOT/terraform" ]]; then
        TF_DIR="{{ROOT}}/modules/$SLOT/terraform"
    elif [[ -d "{{UPSTREAM}}/modules/$SLOT/terraform" ]]; then
        TF_DIR="{{UPSTREAM}}/modules/$SLOT/terraform"
    else
        echo "Error: no terraform root for slot '$SLOT' (expected 'base' or a module with terraform/)." >&2
        exit 1
    fi

    BUCKET=$(uv run {{CFG}} "$CLUSTER" state_bucket)
    if ! aws s3api head-bucket --bucket "${BUCKET}" --region "${STATE_REGION}" 2>/dev/null; then
        echo "ERROR: State bucket '${BUCKET}' does not exist. Run: just bootstrap ${CLUSTER}" >&2
        exit 1
    fi

    echo "━━━ FORCE-UNLOCK: ${CLUSTER}/${SLOT} ━━━"
    echo "  Bucket:   ${BUCKET}"
    echo "  Key:      ${CLUSTER}/${SLOT}/terraform.tfstate"
    echo "  Lock ID:  ${LOCK_ID}"
    echo ""

    cd "$TF_DIR"
    tofu init -reconfigure \
        -backend-config="bucket=${BUCKET}" \
        -backend-config="key=${CLUSTER}/${SLOT}/terraform.tfstate" \
        -backend-config="region=${STATE_REGION}" \
        -backend-config="dynamodb_table=ciforge-terraform-locks"
    tofu force-unlock "${LOCK_ID}"

# Terminate all Karpenter-managed nodes (they will be reprovisioned on demand)
recycle-nodes cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)

    # Ensure kubectl is configured for the target cluster
    just kubeconfig "$CLUSTER"

    echo "Recycling all Karpenter nodes for $CLUSTER ($CNAME)..."

    NODECLAIM_COUNT=$(kubectl get nodeclaims --no-headers 2>/dev/null | wc -l | tr -d ' ')
    if [ "$NODECLAIM_COUNT" = "0" ]; then
        echo "  No Karpenter NodeClaims found — nothing to recycle."
        exit 0
    fi

    echo "  Deleting $NODECLAIM_COUNT NodeClaim(s)..."
    kubectl delete nodeclaims --all --wait=false
    echo "  NodeClaims deleted. Karpenter will reprovision when pods need scheduling."

# Taint ARC runner nodes with NoSchedule (graceful refresh — new jobs go to fresh nodes)
taint-nodes cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)

    # Ensure kubectl is configured for the target cluster
    just kubeconfig "$CLUSTER"

    TAINT_KEY="deploy.osdc.io/refresh-pending"
    TAINT_VALUE="true"
    TAINT_EFFECT="NoSchedule"

    echo "Tainting ARC runner nodes for $CLUSTER ($CNAME)..."
    echo "  Taint: ${TAINT_KEY}=${TAINT_VALUE}:${TAINT_EFFECT}"
    echo ""

    NODES=$(kubectl get nodes -l workload-type=github-runner --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null || true)
    if [ -z "$NODES" ]; then
        echo "  No ARC runner nodes found (workload-type=github-runner) — nothing to taint."
        exit 0
    fi

    TAINTED=0
    SKIPPED=0
    FAILED=0
    for node in $NODES; do
        # Check if taint already exists
        if kubectl get node "$node" -o jsonpath='{.spec.taints[*].key}' 2>/dev/null | tr ' ' '\n' | grep -qx "$TAINT_KEY"; then
            echo "  $node — already tainted, skipping"
            SKIPPED=$((SKIPPED + 1))
        elif kubectl taint nodes "$node" "${TAINT_KEY}=${TAINT_VALUE}:${TAINT_EFFECT}" 2>/dev/null; then
            echo "  $node — tainted"
            TAINTED=$((TAINTED + 1))
        else
            echo "  $node — FAILED (node may have been removed)"
            FAILED=$((FAILED + 1))
        fi
    done

    echo ""
    TOTAL=$((TAINTED + SKIPPED + FAILED))
    echo "Summary: $TAINTED tainted, $SKIPPED already tainted, $FAILED failed, $TOTAL total runner nodes."
    if [ "$FAILED" -gt 0 ]; then
        echo "Warning: $FAILED node(s) failed to taint. Re-run to retry."
    fi

# Remove refresh taint from ARC runner nodes (reversal of taint-nodes)
untaint-nodes cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)

    # Ensure kubectl is configured for the target cluster
    just kubeconfig "$CLUSTER"

    TAINT_KEY="deploy.osdc.io/refresh-pending"
    TAINT_EFFECT="NoSchedule"

    echo "Removing refresh taint from ARC runner nodes for $CLUSTER ($CNAME)..."
    echo "  Taint: ${TAINT_KEY}:${TAINT_EFFECT}"
    echo ""

    NODES=$(kubectl get nodes -l workload-type=github-runner --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null || true)
    if [ -z "$NODES" ]; then
        echo "  No ARC runner nodes found (workload-type=github-runner) — nothing to untaint."
        exit 0
    fi

    REMOVED=0
    SKIPPED=0
    FAILED=0
    for node in $NODES; do
        # Check if taint exists
        if kubectl get node "$node" -o jsonpath='{.spec.taints[*].key}' 2>/dev/null | tr ' ' '\n' | grep -qx "$TAINT_KEY"; then
            if kubectl taint nodes "$node" "${TAINT_KEY}:${TAINT_EFFECT}-" 2>/dev/null; then
                echo "  $node — taint removed"
                REMOVED=$((REMOVED + 1))
            else
                echo "  $node — FAILED (node may have been removed)"
                FAILED=$((FAILED + 1))
            fi
        else
            echo "  $node — not tainted, skipping"
            SKIPPED=$((SKIPPED + 1))
        fi
    done

    echo ""
    TOTAL=$((REMOVED + SKIPPED + FAILED))
    echo "Summary: $REMOVED untainted, $SKIPPED not tainted, $FAILED failed, $TOTAL total runner nodes."
    if [ "$FAILED" -gt 0 ]; then
        echo "Warning: $FAILED node(s) failed to untaint. Re-run to retry."
    fi

# Drain ARC runner scale sets cluster-wide and wait for in-flight runner pods to finish
drain-runners cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)

    # Ensure kubectl is configured for the target cluster
    just kubeconfig "$CLUSTER"

    echo "Draining ARC runner scale sets for $CLUSTER ($CNAME)..."
    echo ""

    ARS_NAMES=$(kubectl get autoscalingrunnersets -n arc-runners --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null || true)
    if [ -z "$ARS_NAMES" ]; then
        echo "  No AutoscalingRunnerSets found in arc-runners — nothing to drain."
        DRAINED=0
        ALREADY=0
        ARS_FAILED=0
    else
        DRAINED=0
        ALREADY=0
        ARS_FAILED=0
        for ars in $ARS_NAMES; do
            CURRENT=$(kubectl get autoscalingrunnerset "$ars" -n arc-runners -o jsonpath='{.spec.maxRunners}' 2>/dev/null || echo "")
            if [ "$CURRENT" = "0" ]; then
                echo "  $ars — already at 0"
                ALREADY=$((ALREADY + 1))
                continue
            fi
            if PATCH_OUT=$(kubectl patch autoscalingrunnerset "$ars" -n arc-runners --type=merge -p '{"spec":{"maxRunners":0}}' 2>&1); then
                echo "  $ars — drained (maxRunners=0)"
                DRAINED=$((DRAINED + 1))
            else
                echo "  $ars — FAILED: ${PATCH_OUT}"
                ARS_FAILED=$((ARS_FAILED + 1))
            fi
        done
    fi

    echo ""
    if [ "${OSDC_TAINT_NODES:-yes}" = "yes" ]; then
        echo "Applying refresh taint to runner nodes..."
        just taint-nodes "$CLUSTER"
        TAINTED_NODES=$(kubectl get nodes -l workload-type=github-runner --no-headers 2>/dev/null | wc -l | tr -d ' ')
        TAINTED_NODES_MSG="${TAINTED_NODES} nodes tainted"
    else
        echo "Skipping refresh taint step (OSDC_TAINT_NODES=${OSDC_TAINT_NODES})."
        TAINTED_NODES_MSG="taint skipped"
    fi

    echo ""
    # Default 1h covers typical PyTorch suite duration; override via OSDC_DRAIN_TIMEOUT_SECS for longer/shorter windows.
    DRAIN_TIMEOUT="${OSDC_DRAIN_TIMEOUT_SECS:-3600}"
    DRAIN_INTERVAL=30
    echo "Waiting for in-flight runner pods to drain (timeout: ${DRAIN_TIMEOUT}s)..."
    SECONDS=0
    DRAIN_DEADLINE=$((SECONDS + DRAIN_TIMEOUT))
    REMAINING=$(kubectl get pods -n arc-runners -l app.kubernetes.io/component=runner --no-headers 2>/dev/null | wc -l | tr -d ' ')
    while [ "$REMAINING" -gt 0 ] && [ "$SECONDS" -lt "$DRAIN_DEADLINE" ]; do
        echo "  ${REMAINING} runner pod(s) still running (elapsed: ${SECONDS}s)..."
        sleep "$DRAIN_INTERVAL"
        REMAINING=$(kubectl get pods -n arc-runners -l app.kubernetes.io/component=runner --no-headers 2>/dev/null | wc -l | tr -d ' ')
    done

    if (( SECONDS < 60 )); then elapsed="${SECONDS}s"; else elapsed="$((SECONDS / 60))m$((SECONDS % 60))s"; fi

    echo ""
    if [ "$REMAINING" -eq 0 ]; then
        echo "All runner pods drained (elapsed: ${elapsed})."
        STRAGGLERS_MSG="all drained"
        EXIT_CODE=0
    else
        echo "Drain timeout reached after ${DRAIN_TIMEOUT}s. Override with OSDC_DRAIN_TIMEOUT_SECS=<seconds> to extend."
        echo "Timed out after ${elapsed} with ${REMAINING} runner pod(s) still running."
        echo "Stragglers:"
        kubectl get pods -n arc-runners -l app.kubernetes.io/component=runner -o wide 2>/dev/null || true
        echo ""
        echo "Operator decision required:"
        echo "  (a) Wait — re-run with OSDC_DRAIN_TIMEOUT_SECS=<larger> to give jobs more time to finish."
        echo "  (b) Force-terminate stragglers: kubectl delete pod -n arc-runners <pod> --grace-period=0 --force"
        echo "      WARNING: this aborts the running GitHub Actions job mid-execution. The job will be"
        echo "      marked failed/timeout on the GitHub side and may auto-retry. The runner registration"
        echo "      may dangle in the GitHub UI until the ARC controller reconciles. Coordinate with"
        echo "      pytorch/pytorch on-call before force-terminating."
        STRAGGLERS_MSG="${REMAINING} stragglers remaining"
        EXIT_CODE=1
    fi

    echo ""
    echo "Summary: ${DRAINED} AutoscalingRunnerSets drained, ${TAINTED_NODES_MSG}, ${STRAGGLERS_MSG}."
    if [ "$ARS_FAILED" -gt 0 ]; then
        echo "Warning: ${ARS_FAILED} AutoscalingRunnerSet(s) failed to patch."
    fi
    if [ "$ALREADY" -gt 0 ]; then
        echo "Note: ${ALREADY} AutoscalingRunnerSet(s) were already at maxRunners=0."
    fi
    exit "$EXIT_CODE"

# Restore maxRunners on every live AutoscalingRunnerSet from def files (out-of-band recovery; cutover uses just deploy)
resume-runners cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)

    # Ensure kubectl is configured for the target cluster
    just kubeconfig "$CLUSTER"

    echo "Resuming ARC runner scale sets for $CLUSTER ($CNAME)..."
    echo ""

    # Build desired-state map { ars-name: maxRunners } from def files
    MAP=$(uv run "{{UPSTREAM}}/modules/arc-runners/scripts/python/runner_max_map.py" "$CLUSTER")

    EXIT_CODE=0
    ARS_NAMES=$(kubectl get autoscalingrunnersets -n arc-runners -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)
    if [ -z "$ARS_NAMES" ]; then
        echo "  No AutoscalingRunnerSets found in arc-runners — nothing to resume."
        RESTORED=0
        ALREADY=0
        WARNINGS=0
        RES_FAILED=0
    else
        RESTORED=0
        ALREADY=0
        WARNINGS=0
        RES_FAILED=0
        for ars in $ARS_NAMES; do
            DESIRED=$(echo "$MAP" | jq -r --arg n "$ars" '.[$n] // empty')
            if [ -z "$DESIRED" ]; then
                echo "  $ars — WARNING: no matching def, leaving as-is"
                WARNINGS=$((WARNINGS + 1))
                EXIT_CODE=1
                continue
            fi
            CURRENT=$(kubectl get autoscalingrunnerset "$ars" -n arc-runners -o jsonpath='{.spec.maxRunners}' 2>/dev/null || echo "")
            if [ "$CURRENT" = "$DESIRED" ]; then
                echo "  $ars — already at $DESIRED"
                ALREADY=$((ALREADY + 1))
                continue
            fi
            if PATCH_OUT=$(kubectl patch autoscalingrunnerset "$ars" -n arc-runners --type=merge -p "{\"spec\":{\"maxRunners\":$DESIRED}}" 2>&1); then
                echo "  $ars — restored maxRunners=${DESIRED}"
                RESTORED=$((RESTORED + 1))
            else
                echo "  $ars — FAILED: ${PATCH_OUT}"
                RES_FAILED=$((RES_FAILED + 1))
                EXIT_CODE=1
            fi
        done
    fi

    echo ""
    if [ "${OSDC_UNTAINT_NODES:-yes}" = "yes" ]; then
        echo "Removing refresh taint from runner nodes..."
        just untaint-nodes "$CLUSTER"
    else
        echo "Skipping untaint step (OSDC_UNTAINT_NODES=${OSDC_UNTAINT_NODES})."
    fi

    echo ""
    echo "Summary: ${RESTORED} restored, ${WARNINGS} warnings, ${RES_FAILED} failures."
    if [ "$ALREADY" -gt 0 ]; then
        echo "Note: ${ALREADY} AutoscalingRunnerSet(s) were already at desired maxRunners."
    fi
    if [ "$EXIT_CODE" -ne 0 ]; then
        echo "Action: investigate failures/warnings above. Run \`just deploy $CLUSTER\` to reconcile from def files."
    fi
    exit "$EXIT_CODE"

# Detect ARC scale-set-id drift (read-only); exits 1 if any drift found
# Note: controller drift is always evaluated globally (not scoped by `name`),
# since a broken controller affects every ARS regardless of which one was named.
heal-arc-check cluster name='':
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    NAME="{{name}}"
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)
    just kubeconfig "$CLUSTER"
    REPORT_PATH=$(just _arc-drift-report "$NAME")
    trap 'rm -f "$REPORT_PATH"' EXIT
    just _arc-drift-print "$REPORT_PATH" "$CNAME"
    DRIFT=$(jq -r '.drift_count // 0' < "$REPORT_PATH" 2>/dev/null || echo "")
    if [ -z "$DRIFT" ] || ! [[ "$DRIFT" =~ ^[0-9]+$ ]]; then
        echo "Error: could not read drift_count from report" >&2
        exit 2
    fi
    [ "$DRIFT" -eq 0 ] && exit 0 || exit 1

# Detect + recover ARC scale-set-id drift (controller restart, re-register, listener/ERS reset)
# Note: controller drift is always evaluated globally (not scoped by `name`),
# since a broken controller affects every ARS regardless of which one was named.
heal-arc cluster name='':
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    NAME="{{name}}"
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)
    just kubeconfig "$CLUSTER"
    REPORT_PATH=$(just _arc-drift-report "$NAME")
    trap 'rm -f "$REPORT_PATH"' EXIT
    just _arc-drift-print "$REPORT_PATH" "$CNAME"
    DRIFT=$(jq -r '.drift_count // 0' < "$REPORT_PATH" 2>/dev/null || echo "")
    if [ -z "$DRIFT" ] || ! [[ "$DRIFT" =~ ^[0-9]+$ ]]; then
        echo "Error: could not read drift_count from report" >&2
        exit 2
    fi
    if [ "$DRIFT" -eq 0 ]; then
        echo "  Nothing to heal."
        exit 0
    fi

    OSDC_CONFIRM="${OSDC_CONFIRM:-ask}"
    case "$OSDC_CONFIRM" in
        yes | true) ;;
        no | false)
            echo "Warning: OSDC_CONFIRM=$OSDC_CONFIRM — aborting without making changes."
            exit 1
            ;;
        *)
            read -r -p "  Heal $DRIFT drifted component(s) on cluster $CNAME? [y/N] " REPLY
            if [[ ! "$REPLY" =~ ^[Yy]$ ]]; then
                echo "  Aborted."
                exit 1
            fi
            ;;
    esac

    FAILED=0
    if [ "$(jq -r '.controller_drift' < "$REPORT_PATH")" = "true" ]; then
        echo "  Controller: restarting arc-gha-rs-controller deployment..."
        if ! kubectl rollout restart deployment/arc-gha-rs-controller -n arc-systems >/dev/null 2>&1; then
            echo "  Controller: FAILED to issue rollout restart — aborting (nothing downstream will reconcile)."
            exit 1
        fi
        if ! kubectl rollout status deployment/arc-gha-rs-controller -n arc-systems --timeout=5m; then
            echo "  Controller: rollout did not become ready within 5m — aborting."
            exit 1
        fi
        echo "  Controller: ready."
    fi

    # ars_rereg = ARSes that need annotation clear + re-registration.
    # Includes both: ARSes missing the annotation, AND ARSes whose listener went
    # missing (forcing the controller to fully reconcile the scale set).
    ARS_REREG=$(jq -r '(.ars_drift + .ars_listener_missing) | unique | .[]' < "$REPORT_PATH")
    REREG_COUNT=0
    REREG_OK=()
    for ars in $ARS_REREG; do
        echo "  ARS $ars: clearing scale-set-id annotation + deleting listener..."
        OLD_ID=$(kubectl get autoscalingrunnerset -n arc-runners "$ars" \
            -o jsonpath="{.metadata.annotations['runner-scale-set-id']}" 2>/dev/null || true)
        if ! kubectl annotate autoscalingrunnerset -n arc-runners "$ars" runner-scale-set-id- >/dev/null 2>&1; then
            echo "  ARS $ars: FAILED to clear annotation"
            FAILED=$((FAILED + 1))
            continue
        fi
        kubectl delete autoscalinglistener -n arc-systems \
            -l "actions.github.com/scale-set-name=$ars,actions.github.com/scale-set-namespace=arc-runners" \
            --ignore-not-found >/dev/null 2>&1 || true
        REREG_COUNT=$((REREG_COUNT + 1))

        # Poll for the controller to stamp a new (different) ID before any
        # ERS cascade — otherwise the recreated ERSes would inherit the OLD
        # ID from the still-stale ARS template.
        NEW_ID=""
        for _ in $(seq 1 30); do
            NEW_ID=$(kubectl get autoscalingrunnerset -n arc-runners "$ars" \
                -o jsonpath="{.metadata.annotations['runner-scale-set-id']}" 2>/dev/null || true)
            if [ -n "$NEW_ID" ] && [ "$NEW_ID" != "$OLD_ID" ]; then
                break
            fi
            sleep 2
        done
        if [ -z "$NEW_ID" ] || [ "$NEW_ID" = "$OLD_ID" ]; then
            echo "  ARS $ars: controller did not stamp a new ID after 60s; skipping ERS cascade"
            FAILED=$((FAILED + 1))
            continue
        fi
        REREG_OK+=("$ars")
    done

    # Build cascade list: ers_drift always cascades; ers_cascade_from_ars only
    # cascades for ARSes that successfully re-registered (to avoid wiping ERSes
    # that would just get recreated with the same stale ID).
    OK_FILTER=""
    if [ "${#REREG_OK[@]}" -gt 0 ]; then
        OK_FILTER=$(printf '%s\n' "${REREG_OK[@]}" | jq -R . | jq -s .)
    else
        OK_FILTER='[]'
    fi

    # Resolve the ERSes that will be deleted in this run. AutoscalingListener
    # and EphemeralRunnerSet are a coupled pair — the listener's
    # spec.ephemeralRunnerSetName is set ONCE at creation and does NOT track
    # ERS renames. Whenever we delete an ERS, its parent ARS's listener MUST
    # also be deleted so the controller recreates them together with consistent
    # name references; otherwise the recreated listener pod crashloops trying
    # to patch the dead ERS name.
    ERS_TO_DELETE=$(jq -r --argjson ok "$OK_FILTER" '
        ((.ers_drift // []) + ((.ers_cascade_from_ars // []) | map(select(.ars as $a | $ok | index($a))) | map(.ers)))
        | unique | .[]' < "$REPORT_PATH")

    # Map ERS names to their parent ARS names so we can cascade listener deletes
    # for any ARS whose ERS is about to be deleted.
    ERS_PARENT_ARS=$(jq -r '
        ((.ers_cascade_from_ars // []) + ((.ers_problems // []) | map({ers: .ers, ars: .ars})))
        | map({key: .ers, value: .ars}) | from_entries' < "$REPORT_PATH")

    # Compute the full set of listeners to delete:
    # - existing listener_drift (id-mismatch, pod-not-ready, etc.)
    # - new listener_orphan_drift (references a nonexistent ERS)
    # - listeners for any ARS whose ERS is about to be deleted (cascade)
    # - listeners for any ARS that is being re-registered
    LISTENER_TO_DELETE=$(jq -r \
        --argjson ok "$OK_FILTER" \
        --argjson ers_parents "$ERS_PARENT_ARS" \
        --arg ers_list "$ERS_TO_DELETE" '
        ($ers_list | split("\n") | map(select(length > 0))) as $ers_del
        | ($ers_del | map($ers_parents[.]) | map(select(. != null and . != ""))) as $ars_from_ers_del
        | (
            (.listener_drift // [])
            + (.listener_orphan_drift // [])
            + (((.listener_problems // []) + (.listener_orphan_problems // []))
                | map(select(.ars as $a | ($ars_from_ers_del + $ok) | index($a)))
                | map(.listener)
                | map(select(. != null)))
          )
        | unique | .[]' < "$REPORT_PATH")

    # Delete listeners FIRST. They take less time than ERS deletes and removing
    # them up front prevents the existing listener pod from crashlooping (and
    # racing the controller) while the new ERS is being created. The controller
    # will recreate both listener and ERS together with consistent references.
    LISTENER_DEL=0
    while IFS= read -r listener; do
        [ -z "$listener" ] && continue
        echo "  Listener $listener: deleting (controller will recreate)..."
        if kubectl delete autoscalinglistener -n arc-systems "$listener" --ignore-not-found >/dev/null 2>&1; then
            LISTENER_DEL=$((LISTENER_DEL + 1))
        else
            echo "  Listener $listener: FAILED to delete"
            FAILED=$((FAILED + 1))
        fi
        # Pace deletes — without this, deleting dozens of listeners at once
        # caused the listener node to exhaust AWS CNI IP allocations as the
        # controller raced to recreate all pods simultaneously.
        sleep 0.5
    done <<< "$LISTENER_TO_DELETE"

    ERS_DEL=0
    while IFS= read -r ers; do
        [ -z "$ers" ] && continue
        echo "  ERS $ers: deleting (cascade-deletes EphemeralRunners)..."
        if kubectl delete ephemeralrunnerset -n arc-runners "$ers" --ignore-not-found >/dev/null 2>&1; then
            ERS_DEL=$((ERS_DEL + 1))
        else
            echo "  ERS $ers: FAILED to delete"
            FAILED=$((FAILED + 1))
        fi
        # Same rationale as the listener loop — pace ERS deletes so the
        # controller's cascade recreation does not stampede the node.
        sleep 0.5
    done <<< "$ERS_TO_DELETE"

    HEALED=$((REREG_COUNT + LISTENER_DEL + ERS_DEL))
    echo ""
    echo "  Healed: $HEALED component(s) | Failed: $FAILED"

    # Verify the recovery actually converged. Sleep briefly to let the
    # controller create new listeners/ERSes, then re-run the drift report.
    # One check after a reasonable settle period — do not wait forever.
    if [ "$FAILED" -eq 0 ] && [ "$HEALED" -gt 0 ]; then
        echo ""
        echo "  Verifying recovery (sleeping 30s for controller to settle)..."
        sleep 30
        VERIFY_PATH=$(just _arc-drift-report "$NAME")
        VERIFY_DRIFT=$(jq -r '.drift_count // 0' < "$VERIFY_PATH" 2>/dev/null || echo "")
        if [ -z "$VERIFY_DRIFT" ] || ! [[ "$VERIFY_DRIFT" =~ ^[0-9]+$ ]]; then
            echo "  Verification: could not read drift_count from post-recovery report"
            rm -f "$VERIFY_PATH"
            FAILED=$((FAILED + 1))
        elif [ "$VERIFY_DRIFT" -eq 0 ]; then
            echo "  Verification: recovery complete (no drift remaining)."
            rm -f "$VERIFY_PATH"
        else
            echo "  Verification: $VERIFY_DRIFT component(s) still drifted after recovery."
            just _arc-drift-print "$VERIFY_PATH" "$CNAME"
            rm -f "$VERIFY_PATH"
            FAILED=$((FAILED + 1))
        fi
    fi

    echo ""
    echo "  Watch progress:"
    echo "    kubectl get autoscalinglisteners -n arc-systems -w"
    echo "    kubectl get ephemeralrunnersets -n arc-runners -w"
    echo ""
    [ "$FAILED" -eq 0 ] && exit 0 || exit 1

# Build ARC drift report as JSON (internal helper for heal-arc / heal-arc-check).
# Writes the report to a tempfile under /tmp and echoes ONLY the path on stdout.
# Caller is responsible for deleting the tempfile.
_arc-drift-report name='':
    #!/usr/bin/env bash
    set -euo pipefail
    NAME="{{name}}"

    # Stage kubectl outputs in tempfiles — combined JSON exceeds ARG_MAX on
    # large clusters (~1.25MB ARS blob alone on meta-staging-aws-uw1), so --argjson is
    # not viable. --slurpfile takes a file path instead.
    WORK=$(mktemp -d)
    trap 'rm -rf "$WORK"' EXIT
    CTRL_F="$WORK/ctrl.json"
    ARS_F="$WORK/ars.json"
    LISTENERS_F="$WORK/listeners.json"
    PODS_F="$WORK/pods.json"
    ERS_F="$WORK/ers.json"

    # Controller: query by deployment name (not label) — same source-of-truth
    # for both detection and recovery. --ignore-not-found returns empty body
    # rather than an error if the deployment is absent.
    kubectl get deployment -n arc-systems arc-gha-rs-controller -o json --ignore-not-found 2>/dev/null > "$CTRL_F" || true
    # Wrap the single-deployment response in a List shape ({items: [...]})
    # so downstream jq can treat it uniformly with the other queries.
    if [ ! -s "$CTRL_F" ]; then
        echo '{"items":[]}' > "$CTRL_F"
    elif ! jq -e '.items' < "$CTRL_F" >/dev/null 2>&1; then
        jq '{items: [.]}' < "$CTRL_F" > "$CTRL_F.tmp" && mv "$CTRL_F.tmp" "$CTRL_F"
    fi

    kubectl get autoscalingrunnersets -n arc-runners -o json 2>/dev/null > "$ARS_F" || echo '{"items":[]}' > "$ARS_F"
    if [ -n "$NAME" ]; then
        jq --arg n "$NAME" '.items |= map(select(.metadata.name == $n))' < "$ARS_F" > "$ARS_F.tmp" \
            && mv "$ARS_F.tmp" "$ARS_F"
    fi
    kubectl get autoscalinglisteners -n arc-systems -o json 2>/dev/null > "$LISTENERS_F" || echo '{"items":[]}' > "$LISTENERS_F"
    # Listener pods: queried unfiltered, looked up by NAME (== AutoscalingListener
    # resource name, per ARC's resourcebuilder.go). Avoids guessing labels.
    kubectl get pods -n arc-systems -o json 2>/dev/null > "$PODS_F" || echo '{"items":[]}' > "$PODS_F"
    kubectl get ephemeralrunnersets -n arc-runners -o json 2>/dev/null > "$ERS_F" || echo '{"items":[]}' > "$ERS_F"

    REPORT_PATH=$(mktemp -t heal-arc-report.XXXXXX.json)

    jq -n \
        --slurpfile ctrl "$CTRL_F" \
        --slurpfile ars "$ARS_F" \
        --slurpfile listeners "$LISTENERS_F" \
        --slurpfile pods "$PODS_F" \
        --slurpfile ers "$ERS_F" '
        ($ctrl[0]) as $ctrl
        | ($ars[0]) as $ars
        | ($listeners[0]) as $listeners
        | ($pods[0]) as $pods
        | ($ers[0]) as $ers

        # Build a name -> pod index once so per-listener lookups are O(1).
        | (($pods.items // []) | map({key: .metadata.name, value: .}) | from_entries) as $pod_by_name

        | def pod_status($name):
            ($pod_by_name[$name] // null)
            | if . == null then "Missing"
              elif (.status.phase != "Running") then .status.phase
              elif (([.status.containerStatuses[]? | select(.ready == true)] | length) == 0) then "NotReady"
              else "Ready" end;

        ($ctrl.items // []) as $cdeps
        | (if ($cdeps | length) == 0 then true
           else ($cdeps | any((.status.readyReplicas // 0) < (.spec.replicas // 0))) end) as $controller_drift

        | ($ars.items // []) as $arsl
        | [ $arsl[] | select((.metadata.annotations["runner-scale-set-id"] // "") == "") | .metadata.name ] as $ars_missing_id

        | [ $arsl[] | select((.metadata.annotations["runner-scale-set-id"] // "") != "")
            | . as $a
            | (.metadata.annotations["runner-scale-set-id"] | tonumber) as $cid
            | ($listeners.items // []) | map(select(
                .spec.autoscalingRunnerSetName == $a.metadata.name
                and .spec.autoscalingRunnerSetNamespace == "arc-runners"
              )) | first as $l
            | if $l == null then { ars: $a.metadata.name, listener: null, reason: "missing" }
              elif ($l.spec.runnerScaleSetId != $cid) then
                { ars: $a.metadata.name, listener: $l.metadata.name, reason: ("id-mismatch (\($l.spec.runnerScaleSetId) != \($cid))") }
              else (pod_status($l.metadata.name)) as $st
                | if $st == "Ready" then empty
                  else { ars: $a.metadata.name, listener: $l.metadata.name, reason: ("pod-\($st)") } end
              end
          ] as $listener_problems

        # ARSes whose listener resource is entirely missing — these need the
        # parent ARS re-registered (annotation clear), not just a listener delete.
        | [ $listener_problems[] | select(.listener == null) | .ars ] as $ars_listener_missing

        | [ $ers.items[]
            | . as $e
            | ((.metadata.ownerReferences // []) | map(select(.kind == "AutoscalingRunnerSet")) | first) as $owner
            | if $owner == null then empty
              else $arsl | map(select(.metadata.name == $owner.name)) | first as $owning_ars
                | if $owning_ars == null then empty
                  else ($owning_ars.metadata.annotations["runner-scale-set-id"] // "") as $cidstr
                    | if $cidstr == "" then empty
                      else ($cidstr | tonumber) as $cid
                        # Skip ERSes whose runnerScaleSetId is unset — they are
                        # in-flight being created, not stale.
                        | if $e.spec.ephemeralRunnerSpec.runnerScaleSetId == null then empty
                          elif $e.spec.ephemeralRunnerSpec.runnerScaleSetId != $cid then
                            { ers: $e.metadata.name, ars: $owner.name, expected: $cid, actual: $e.spec.ephemeralRunnerSpec.runnerScaleSetId }
                          else empty end
                      end
                  end
              end
          ] as $ers_problems

        # ERSes whose parent ARS will be re-registered (annotation cleared) —
        # they need to be cascade-deleted after re-registration completes,
        # because the new ID will not propagate to existing ERS templates.
        # Pairs are (ers, ars) so the heal loop can filter by which ARSes
        # actually re-registered successfully.
        | [ $ers.items[]
            | . as $e
            | ((.metadata.ownerReferences // []) | map(select(.kind == "AutoscalingRunnerSet")) | first) as $owner
            | select($owner != null and (($ars_missing_id + $ars_listener_missing) | index($owner.name)))
            | { ers: $e.metadata.name, ars: $owner.name }
          ] as $ers_cascade

        # Build the set of existing ERS names for orphan lookup.
        | (($ers.items // []) | map(.metadata.name)) as $ers_names

        # Listeners whose spec.ephemeralRunnerSetName references an ERS that
        # no longer exists. The listener spec is set ONCE at creation; ARS
        # re-registration creates a NEW ERS with a fresh random suffix, but the
        # old listener spec still points at the old (now-deleted) ERS name. The
        # listener pod then crashloops trying to patch the dead ERS. Recovery:
        # delete the listener so the controller recreates it with the current
        # ERS name embedded.
        | [ ($listeners.items // [])[]
            | . as $l
            | ($l.spec.ephemeralRunnerSetName // "") as $target
            | select($target != "" and (($ers_names | index($target)) | not))
            | { listener: $l.metadata.name, ars: ($l.spec.autoscalingRunnerSetName // ""), stale_ers: $target }
          ] as $listener_orphan_problems

        | {
            controller_drift: $controller_drift,
            ars_drift: $ars_missing_id,
            ars_listener_missing: $ars_listener_missing,
            listener_drift: ($listener_problems | map(select(.listener != null)) | map(.listener)),
            listener_problems: $listener_problems,
            listener_orphan_drift: ($listener_orphan_problems | map(.listener)),
            listener_orphan_problems: $listener_orphan_problems,
            ers_drift: ($ers_problems | map(.ers)),
            ers_problems: $ers_problems,
            ers_cascade_from_ars: $ers_cascade,
            drift_count: (
                (if $controller_drift then 1 else 0 end)
                + ($ars_missing_id | length)
                + ($listener_problems | length)
                + ($listener_orphan_problems | length)
                + ($ers_problems | length)
            )
          }
        ' > "$REPORT_PATH"

    echo "$REPORT_PATH"

# Pretty-print ARC drift report (internal helper for heal-arc / heal-arc-check).
# Takes the path to the report tempfile (not the JSON itself) to keep arg sizes small.
_arc-drift-print report_path cname:
    #!/usr/bin/env bash
    set -euo pipefail
    REPORT_PATH="{{report_path}}"
    CNAME="{{cname}}"
    if [ ! -s "$REPORT_PATH" ]; then
        echo "Error: report file is empty or missing: $REPORT_PATH" >&2
        exit 2
    fi
    echo "ARC drift report for cluster $CNAME:"
    echo ""
    if [ "$(jq -r '.controller_drift' < "$REPORT_PATH")" = "true" ]; then
        echo "  Controller (arc-gha-rs-controller): DRIFT (not ready or missing)"
    else
        echo "  Controller (arc-gha-rs-controller): OK"
    fi
    echo ""
    echo "  ARS missing scale-set-id annotation:"
    if [ "$(jq -r '.ars_drift | length' < "$REPORT_PATH")" -eq 0 ]; then
        echo "    (none)"
    else
        jq -r '.ars_drift[] | "    - \(.)"' < "$REPORT_PATH"
    fi
    echo ""
    echo "  Listener drift (missing / id-mismatch / pod-not-ready):"
    if [ "$(jq -r '.listener_problems | length' < "$REPORT_PATH")" -eq 0 ]; then
        echo "    (none)"
    else
        jq -r '.listener_problems[] | "    - ARS \(.ars): \(.reason)"' < "$REPORT_PATH"
    fi
    echo ""
    echo "  Listener orphan (references nonexistent ERS):"
    if [ "$(jq -r '.listener_orphan_problems | length' < "$REPORT_PATH")" -eq 0 ]; then
        echo "    (none)"
    else
        jq -r '.listener_orphan_problems[] | "    - \(.listener): refers to ERS \(.stale_ers) (does not exist)"' < "$REPORT_PATH"
    fi
    echo ""
    echo "  ERS drift (cached runnerScaleSetId mismatches parent ARS):"
    if [ "$(jq -r '.ers_problems | length' < "$REPORT_PATH")" -eq 0 ]; then
        echo "    (none)"
    else
        jq -r '.ers_problems[] | "    - \(.ers): expected \(.expected), actual \(.actual) (parent ARS: \(.ars))"' < "$REPORT_PATH"
    fi
    CASCADE=$(jq -r '.ers_cascade_from_ars | length' < "$REPORT_PATH")
    if [ "$CASCADE" -gt 0 ]; then
        echo ""
        echo "  ERS cascade-delete queued (parent ARS needs re-registration):"
        jq -r '.ers_cascade_from_ars[] | "    - \(.ers) (ARS: \(.ars))"' < "$REPORT_PATH"
    fi
    echo ""
    DRIFT=$(jq -r '.drift_count' < "$REPORT_PATH")
    if [ "$DRIFT" -eq 0 ]; then
        echo "  Summary: no drift detected."
    else
        echo "  Summary: $DRIFT component(s) drifted."
    fi

# ============================================================================
# INTERNAL: Base sub-deploys
# ============================================================================

# Deploy Harbor (called from deploy-base)
_deploy-harbor cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    source "{{UPSTREAM}}/scripts/helm-upgrade.sh"
    source "{{UPSTREAM}}/scripts/state-config.sh"
    : "${STATE_REGION:?state-config.sh did not export STATE_REGION}"
    source "{{UPSTREAM}}/scripts/kubectl-apply.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    CNAME=$(uv run {{CFG}} "$CLUSTER" cluster_name)
    REGION=$(uv run {{CFG}} "$CLUSTER" region)
    BUCKET=$(uv run {{CFG}} "$CLUSTER" state_bucket)

    # Ensure kubectl is configured for the target cluster
    just kubeconfig "$CLUSTER"

    echo ""
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo "⎈  Harbor Pull-Through Cache - ${CLUSTER}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    # Harbor is the first k8s workload deployed — wait for base nodes
    echo "Waiting for base nodes to be ready..."
    kubectl wait --for=condition=Ready nodes -l role=base-infrastructure --timeout=10m
    echo ""

    # Get Terraform outputs (init with cluster-specific backend)
    cd {{UPSTREAM}}/modules/eks/terraform
    tofu init -reconfigure \
        -backend-config="bucket=${BUCKET}" \
        -backend-config="key=${CLUSTER}/base/terraform.tfstate" \
        -backend-config="region=${STATE_REGION}" \
        -backend-config="dynamodb_table=ciforge-terraform-locks" \
        >/dev/null 2>&1
    HARBOR_ROLE=$(tofu output -raw harbor_role_arn)
    HARBOR_BUCKET=$(tofu output -raw harbor_s3_bucket)
    HARBOR_S3_KEY=$(tofu output -raw harbor_s3_access_key_id)
    HARBOR_S3_SECRET=$(tofu output -raw harbor_s3_secret_access_key)
    AWS_REGION=$(tofu output -raw aws_region 2>/dev/null || echo "us-west-2")
    AWS_ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
    cd - >/dev/null
    echo ""

    ECR_REGISTRY="${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com"

    # Create namespace (needed before creating secrets/service accounts)
    kubectl create namespace harbor-system 2>/dev/null || true

    # Create ServiceAccount for Harbor registry (IRSA)
    # The Harbor chart only sets serviceAccountName on the pod spec but does NOT create the SA.
    echo "Creating harbor-registry ServiceAccount..."
    kubectl apply -f - <<EOF
    apiVersion: v1
    kind: ServiceAccount
    metadata:
        name: harbor-registry
        namespace: harbor-system
        annotations:
            eks.amazonaws.com/role-arn: "${HARBOR_ROLE}"
    EOF

    # Create S3 credentials secret for Harbor registry
    # The goharbor/distribution S3 driver does not support IRSA (web identity tokens),
    # so we provide static IAM credentials via a Kubernetes Secret.
    echo "Creating Harbor S3 credentials secret..."
    kubectl create secret generic harbor-s3-credentials \
        --namespace harbor-system \
        --from-literal=REGISTRY_STORAGE_S3_ACCESSKEY="${HARBOR_S3_KEY}" \
        --from-literal=REGISTRY_STORAGE_S3_SECRETKEY="${HARBOR_S3_SECRET}" \
        --dry-run=client -o yaml | kubectl apply -f -
    echo ""

    # Auto-generate Harbor admin password (once, on first deploy)
    if ! kubectl get secret harbor-admin-password -n harbor-system >/dev/null 2>&1; then
        echo "Generating Harbor admin password..."
        HARBOR_ADMIN_PW=$(openssl rand -base64 32 | tr -d '/+=' | head -c 32)
        kubectl create secret generic harbor-admin-password \
            --namespace harbor-system \
            --from-literal=password="${HARBOR_ADMIN_PW}"
    else
        echo "Harbor admin password secret already exists, reusing..."
    fi
    HARBOR_ADMIN_PW=$(kubectl get secret harbor-admin-password -n harbor-system \
        -o jsonpath='{.data.password}' | base64 -d)

    # Auto-generate Harbor internal DB password (once, on first deploy)
    if ! kubectl get secret harbor-db-password -n harbor-system >/dev/null 2>&1; then
        echo "Generating Harbor DB password..."
        HARBOR_DB_PW=$(openssl rand -base64 32 | tr -d '/+=' | head -c 32)
        kubectl create secret generic harbor-db-password \
            --namespace harbor-system \
            --from-literal=password="${HARBOR_DB_PW}"
    else
        echo "Harbor DB password secret already exists, reusing..."
    fi
    HARBOR_DB_PW=$(kubectl get secret harbor-db-password -n harbor-system \
        -o jsonpath='{.data.password}' | base64 -d)
    echo ""

    # Read per-cluster Harbor config
    HARBOR_CORE_REPLICAS=$(uv run {{CFG}} "$CLUSTER" harbor.core_replicas 2)
    HARBOR_REGISTRY_REPLICAS=$(uv run {{CFG}} "$CLUSTER" harbor.registry_replicas 2)
    HARBOR_NGINX_REPLICAS=$(uv run {{CFG}} "$CLUSTER" harbor.nginx_replicas 3)
    HARBOR_PDB_MAX_UNAVAILABLE=$(uv run {{CFG}} "$CLUSTER" harbor.pdb_max_unavailable 1)

    # Validate: positive integer or "1%"-"100%". Rejects 0/"0%" (deadlocks
    # all evictions), empty strings (renders broken YAML), and any value
    # containing sed/shell metacharacters that would corrupt the substitution.
    if [[ ! "$HARBOR_PDB_MAX_UNAVAILABLE" =~ ^([1-9][0-9]*|[1-9][0-9]?%|100%)$ ]]; then
        echo "ERROR: harbor.pdb_max_unavailable='${HARBOR_PDB_MAX_UNAVAILABLE}' is invalid." >&2
        echo "       Expected: positive integer (e.g., 1, 2, 5) or percentage '1%'-'100%' (quoted)." >&2
        exit 1
    fi

    # Install Harbor (using traditional Helm repo to avoid ghcr.io auth/rate-limit issues)
    echo "Installing Harbor..."
    helm repo add harbor https://helm.goharbor.io 2>/dev/null || true
    helm repo update harbor
    # Override each component's image to use ECR mirror
    # (Harbor chart has no global.imageRegistry; each component sets image.repository individually)
    ECR_HARBOR="${ECR_REGISTRY}/mirror/goharbor"

    # Use input-hash strategy for Harbor: the chart uses randAlphaNum/genCA
    # which produce non-deterministic template output, making template-based
    # comparison unreliable. Input hashing compares what we send to Helm,
    # not what Helm renders — immune to random secret generation.
    HELM_MAX_RETRIES=3
    for attempt in $(seq 1 $HELM_MAX_RETRIES); do
        if helm_upgrade_by_input_hash harbor harbor-system \
            --create-namespace \
            --history-max 3 \
            -f {{UPSTREAM}}/base/helm/harbor/values.yaml \
            --set core.image.repository="${ECR_HARBOR}/harbor-core" \
            --set jobservice.image.repository="${ECR_HARBOR}/harbor-jobservice" \
            --set portal.image.repository="${ECR_HARBOR}/harbor-portal" \
            --set registry.registry.image.repository="${ECR_HARBOR}/registry-photon" \
            --set registry.controller.image.repository="${ECR_HARBOR}/harbor-registryctl" \
            --set database.internal.image.repository="${ECR_HARBOR}/harbor-db" \
            --set redis.internal.image.repository="${ECR_HARBOR}/redis-photon" \
            --set nginx.image.repository="${ECR_HARBOR}/nginx-photon" \
            --set exporter.image.repository="${ECR_HARBOR}/harbor-exporter" \
            --set persistence.imageChartStorage.s3.bucket="${HARBOR_BUCKET}" \
            --set persistence.imageChartStorage.s3.region="${AWS_REGION}" \
            --set persistence.imageChartStorage.s3.regionendpoint="https://s3.dualstack.${AWS_REGION}.amazonaws.com" \
            --set persistence.imageChartStorage.s3.existingSecret=harbor-s3-credentials \
            --set registry.serviceAccountName=harbor-registry \
            --set registry.automountServiceAccountToken=true \
            --set harborAdminPassword="${HARBOR_ADMIN_PW}" \
            --set database.internal.password="${HARBOR_DB_PW}" \
            --set core.replicas="${HARBOR_CORE_REPLICAS}" \
            --set registry.replicas="${HARBOR_REGISTRY_REPLICAS}" \
            --set nginx.replicas="${HARBOR_NGINX_REPLICAS}" \
            --timeout 15m \
            --wait \
            --burst-limit 10000 \
            --qps 500 \
            harbor/harbor \
            --version 1.18.2; then
            break
        fi
        if [ "$attempt" -eq "$HELM_MAX_RETRIES" ]; then
            echo "Helm upgrade failed after $HELM_MAX_RETRIES attempts"
            exit 1
        fi
        echo "  Helm upgrade attempt $attempt/$HELM_MAX_RETRIES failed, retrying in 30s..."
        sleep 30
    done
    echo ""

    # Apply Harbor PodDisruptionBudgets (chart has no native PDB support).
    # Generated from base/kubernetes/harbor/pdb.yaml.tpl with maxUnavailable
    # substituted from clusters.yaml -> harbor.pdb_max_unavailable.
    echo "Applying Harbor PodDisruptionBudgets..."
    GENERATED_HARBOR_PDB=$(mktemp)
    trap 'rm -f "$GENERATED_HARBOR_PDB"' EXIT
    sed -e "s|__MAX_UNAVAILABLE__|${HARBOR_PDB_MAX_UNAVAILABLE}|g" \
        "{{UPSTREAM}}/base/kubernetes/harbor/pdb.yaml.tpl" >"$GENERATED_HARBOR_PDB"
    kubectl_apply_if_changed -f "$GENERATED_HARBOR_PDB"
    echo ""

    # Ensure Harbor admin password matches the K8s secret.
    # harborAdminPassword only takes effect on first install; on upgrades the
    # DB may still hold the old default. Always attempt migration — it's
    # idempotent (401 = already migrated, 200 = just migrated).
    echo "Ensuring Harbor admin password is up to date..."
    kubectl port-forward --address 127.0.0.1 -n harbor-system svc/harbor 8090:80 &
    PW_PF_PID=$!
    PW_MAX_RETRIES=12
    for pw_attempt in $(seq 1 $PW_MAX_RETRIES); do
        if curl -sf -o /dev/null http://localhost:8090/api/v2.0/health 2>/dev/null; then
            break
        fi
        if [ "$pw_attempt" -eq "$PW_MAX_RETRIES" ]; then
            echo "  Harbor API not reachable after $PW_MAX_RETRIES attempts"
            kill $PW_PF_PID 2>/dev/null || true
            exit 1
        fi
        echo "  Waiting for Harbor API (attempt $pw_attempt/$PW_MAX_RETRIES)..."
        sleep 5
    done
    HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
        -X PUT "http://localhost:8090/api/v2.0/users/1/password" \
        -u "admin:Harbor12345" \
        -H "Content-Type: application/json" \
        -d "{\"old_password\":\"Harbor12345\",\"new_password\":\"${HARBOR_ADMIN_PW}\"}")
    kill $PW_PF_PID 2>/dev/null || true
    if [ "$HTTP_CODE" = "200" ]; then
        echo "  Admin password migrated from default"
    elif [ "$HTTP_CODE" = "401" ]; then
        echo "  Password already up to date"
    else
        echo "  Warning: password migration returned HTTP $HTTP_CODE"
    fi
    echo ""

    # Configure proxy cache projects
    echo "Configuring proxy cache projects..."
    just _configure-harbor-projects
    echo ""
    echo "Harbor deployed."

# Configure Harbor proxy cache projects via REST API (called from _deploy-harbor)
_configure-harbor-projects:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    echo "Configuring Harbor proxy cache projects..."

    # Read admin password from Kubernetes secret
    HARBOR_ADMIN_PW=$(kubectl get secret harbor-admin-password -n harbor-system \
        -o jsonpath='{.data.password}' | base64 -d)

    # Read upstream registry credentials (optional — secrets may not exist)
    CRED_ARGS=""
    if kubectl get secret harbor-dockerhub-credentials -n harbor-system >/dev/null 2>&1; then
        DH_USER=$(kubectl get secret harbor-dockerhub-credentials -n harbor-system \
            -o jsonpath='{.data.username}' | base64 -d)
        DH_TOKEN=$(kubectl get secret harbor-dockerhub-credentials -n harbor-system \
            -o jsonpath='{.data.token}' | base64 -d)
        CRED_ARGS="${CRED_ARGS} --dockerhub-username ${DH_USER} --dockerhub-token ${DH_TOKEN}"
        echo "  Docker Hub credentials found"
    fi
    if kubectl get secret harbor-github-credentials -n harbor-system >/dev/null 2>&1; then
        GH_USER=$(kubectl get secret harbor-github-credentials -n harbor-system \
            -o jsonpath='{.data.username}' | base64 -d)
        GH_TOKEN=$(kubectl get secret harbor-github-credentials -n harbor-system \
            -o jsonpath='{.data.token}' | base64 -d)
        CRED_ARGS="${CRED_ARGS} --github-username ${GH_USER} --github-token ${GH_TOKEN}"
        echo "  GitHub credentials found"
    fi

    # Port-forward to Harbor for API access
    kubectl port-forward --address 127.0.0.1 -n harbor-system svc/harbor 8080:80 &
    PF_PID=$!
    trap "kill $PF_PID 2>/dev/null || true" EXIT
    HP_MAX_RETRIES=12
    for hp_attempt in $(seq 1 $HP_MAX_RETRIES); do
        if curl -sf -o /dev/null http://localhost:8080/api/v2.0/health 2>/dev/null; then
            break
        fi
        if [ "$hp_attempt" -eq "$HP_MAX_RETRIES" ]; then
            echo "  Harbor API not reachable after $HP_MAX_RETRIES attempts"
            exit 1
        fi
        echo "  Waiting for Harbor API (attempt $hp_attempt/$HP_MAX_RETRIES)..."
        sleep 5
    done

    uv run {{SCRIPTS}}/python/configure_harbor_projects.py \
        --harbor-url http://localhost:8080 \
        --admin-password "${HARBOR_ADMIN_PW}" \
        ${CRED_ARGS}

    kill $PF_PID 2>/dev/null || true

# ============================================================================
# TESTING
# ============================================================================

# Run all tests across the project
test:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    SECONDS=0

    # Agent mode: suppress output on success, show only on failure
    _AGENT="${AGENT_ENVIRONMENT:-false}"
    _agent_out=""
    if [[ "$_AGENT" == "true" ]]; then
        _agent_out=$(mktemp)
        exec 3>&1 4>&2
        exec > "$_agent_out" 2>&1
    fi
    trap '[[ -n "$_agent_out" ]] && rm -f "$_agent_out"' EXIT

    # Find all directories containing test_*.py files. Excludes:
    #   - tests/e2e (live-cluster end-to-end tests)
    #   - base/<component>/tests/smoke and modules/<module>/tests/smoke
    #     (live-cluster smoke tests — run via `just smoke <cluster>`)
    # The project-root tests/smoke/ is intentionally INCLUDED — it holds unit
    # tests for the smoke helpers package (test_helpers.py, helpers/test_retry.py)
    # which exercise pure logic and need to run in `just test`.
    TEST_DIRS=$(find "{{UPSTREAM}}" -path '*/.scratch' -prune -o -path '*/.venv' -prune -o -path '*/tests/e2e' -prune -o -path '*/base/*/tests/smoke' -prune -o -path '*/modules/*/tests/smoke' -prune -o -name 'test_*.py' -exec dirname {} \; | sort -u)
    if [[ "{{ROOT}}" != "{{UPSTREAM}}" ]]; then
        TEST_DIRS="$TEST_DIRS $(find "{{ROOT}}" -path "{{UPSTREAM}}" -prune -o -path '*/.scratch' -prune -o -path '*/.venv' -prune -o -path '*/tests/e2e' -prune -o -path '*/base/*/tests/smoke' -prune -o -path '*/modules/*/tests/smoke' -prune -o -name 'test_*.py' -exec dirname {} \; | sort -u)"
    fi

    if [[ -z "$TEST_DIRS" ]]; then
        if [[ "$_AGENT" == "true" ]]; then exec 1>&3 2>&4; echo "OK (${SECONDS}s)"; rm -f "$_agent_out"; exit 0; fi
        echo "No test files found."
        exit 0
    fi

    # Build --cov flags for each source directory
    # tests/smoke/ and its helpers/ subpackage hold unit tests that exercise
    # selective parts of the smoke helpers — full coverage is provided by the
    # live-cluster smoke runs (`just smoke`). Exclude them from the per-file
    # coverage gate so that the unit tests can run without forcing 95%
    # coverage on every helper file.
    echo "Test directories:"
    COV_FLAGS=""
    for dir in $TEST_DIRS; do
        echo "  - ${dir#{{UPSTREAM}}/}"
        case "$dir" in
            */tests/smoke|*/tests/smoke/*) ;;
            *) COV_FLAGS="$COV_FLAGS --cov=$dir" ;;
        esac
    done
    echo ""

    # Run all tests in parallel (pytest-xdist -n auto), single invocation
    # Same pattern as 'just smoke' — xdist distributes tests across CPU cores
    uv run pytest $TEST_DIRS \
        -n auto \
        --tb=short -q \
        $COV_FLAGS \
        --cov-config="{{UPSTREAM}}/pyproject.toml" \
        --cov-report=term-missing:skip-covered \
        --cov-report=json \
        --rootdir="{{UPSTREAM}}" \
        && rc=0 || rc=$?

    # Per-file coverage check (only if pytest passed)
    if [[ $rc -eq 0 && -f coverage.json ]]; then
        python3 << 'COVERAGE_CHECK' || rc=$?
    import json, sys
    with open("coverage.json") as f:
        data = json.load(f)
    threshold = 95
    failures = []
    for fname, fdata in sorted(data.get("files", {}).items()):
        pct = fdata["summary"]["percent_covered"]
        if pct < threshold:
            failures.append((fname, pct))
    if failures:
        print(f"Per-file coverage below {threshold}%:")
        for fname, pct in failures:
            print(f"  {fname}: {pct:.0f}%")
        sys.exit(1)
    COVERAGE_CHECK
    fi

    rm -f coverage.json .coverage

    if (( SECONDS < 60 )); then elapsed="${SECONDS}s"; else elapsed="$((SECONDS / 60))m$((SECONDS % 60))s"; fi
    if [[ "$_AGENT" == "true" ]]; then
        exec 1>&3 2>&4
        if [[ $rc -eq 0 ]]; then echo "OK (${elapsed})"; else cat "$_agent_out"; fi
        rm -f "$_agent_out"
        exit $rc
    fi
    if [[ $rc -ne 0 ]]; then
        echo "Tests FAILED. (${elapsed})"
        exit $rc
    fi
    echo "All tests passed. (${elapsed})"

# Run node-compactor e2e tests against a live cluster
test-compactor cluster:
    @just kubeconfig {{cluster}}
    @export CLUSTERS_YAML="{{CLUSTERS_YAML}}"; \
    export OSDC_ROOT="{{ROOT}}"; \
    export OSDC_UPSTREAM="{{UPSTREAM}}"; \
    SECONDS=0; \
    cd {{UPSTREAM}}/base/node-compactor/tests/e2e && \
    uv run pytest test_e2e.py -v --tb=long \
        --cluster-id="{{cluster}}" \
        && rc=0 || rc=$?; \
    echo ""; \
    if (( SECONDS < 60 )); then elapsed="${SECONDS}s"; else elapsed="$((SECONDS / 60))m$((SECONDS % 60))s"; fi; \
    echo "Compactor e2e tests completed in ${elapsed}"; \
    exit $rc

# Run image-cache-janitor e2e tests against a live cluster
test-janitor cluster:
    @just kubeconfig {{cluster}}
    @export CLUSTERS_YAML="{{CLUSTERS_YAML}}"; \
    export OSDC_ROOT="{{ROOT}}"; \
    export OSDC_UPSTREAM="{{UPSTREAM}}"; \
    SECONDS=0; \
    cd {{UPSTREAM}}/base/kubernetes/image-cache-janitor/tests/e2e && \
    uv run pytest test_e2e.py -v --tb=long \
        --cluster-id="{{cluster}}" \
        && rc=0 || rc=$?; \
    echo ""; \
    if (( SECONDS < 60 )); then elapsed="${SECONDS}s"; else elapsed="$((SECONDS / 60))m$((SECONDS % 60))s"; fi; \
    echo "Janitor e2e tests completed in ${elapsed}"; \
    exit $rc

# Run smoke tests against a deployed cluster
smoke cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"

    # Agent mode: suppress output on success, show only on failure
    _AGENT="${AGENT_ENVIRONMENT:-false}"
    _agent_out=""
    if [[ "$_AGENT" == "true" ]]; then
        _agent_out=$(mktemp)
        exec 3>&1 4>&2
        exec > "$_agent_out" 2>&1
    fi
    trap '[[ -n "$_agent_out" ]] && rm -f "$_agent_out"' EXIT

    UPSTREAM="{{UPSTREAM}}"
    ROOT="{{ROOT}}"
    CLUSTER="{{cluster}}"

    # Ensure kubectl is configured for the target cluster
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    just kubeconfig "$CLUSTER"

    # Discover base smoke tests (co-located with each base component)
    # Also includes modules/eks/ which is always deployed as part of base
    SMOKE_DIRS=""
    while IFS= read -r d; do
        SMOKE_DIRS="${SMOKE_DIRS:+$SMOKE_DIRS }$d"
    done < <(find "${UPSTREAM}/base" "${UPSTREAM}/modules/eks" -path '*/tests/smoke' -type d 2>/dev/null | sort)

    # Add smoke tests for each enabled module
    MODULES=$(uv run "${UPSTREAM}/scripts/cluster-config.py" "$CLUSTER" modules)
    for mod in $MODULES; do
        # Check consumer modules first (ROOT), then upstream
        if [[ -d "${ROOT}/modules/${mod}/tests/smoke" ]]; then
            SMOKE_DIRS="$SMOKE_DIRS ${ROOT}/modules/${mod}/tests/smoke"
        elif [[ -d "${UPSTREAM}/modules/${mod}/tests/smoke" ]]; then
            SMOKE_DIRS="$SMOKE_DIRS ${UPSTREAM}/modules/${mod}/tests/smoke"
        fi
    done

    echo "Running smoke tests for cluster: $CLUSTER"
    echo "Test directories:"
    for d in $SMOKE_DIRS; do echo "  - ${d#${UPSTREAM}/}"; done
    echo ""

    # Pre-generate ARC runner YAMLs for every enabled arc-runners* module
    # (canonical + per-GPU-arch variants like arc-runners-b200 / arc-runners-h100).
    # Each variant has its own defs/ and generated/ but shares the canonical
    # generator (parameterized via ARC_RUNNERS_* env vars — same pattern as
    # each variant's own deploy.sh, e.g. modules/arc-runners-b200/deploy.sh).
    # Generation must happen ONCE before any pytest worker starts —
    # previously the test fixture shelled out, which raced under pytest-xdist.
    # OSDC_RESOLVER_READONLY: smoke must not mutate the lock ConfigMap or hit GitHub.
    # Prefers an exact match for the current osdc SHA; falls back to the newest
    # entry when the current SHA isn't in the lock (e.g. when iterating on test
    # code without a fresh deploy).
    export OSDC_RESOLVER_READONLY=1
    for mod in $MODULES; do
        case "$mod" in
            arc-runners | arc-runners-*)
                echo "── Pre-generating YAMLs for $mod (test prerequisite) ──"
                ARC_RUNNERS_DEFS_DIR="$UPSTREAM/modules/$mod/defs" \
                    ARC_RUNNERS_OUTPUT_DIR="$UPSTREAM/modules/$mod/generated" \
                    ARC_RUNNERS_MODULE_NAME="$mod" \
                    just generate-arc-runners "$CLUSTER"
                echo ""
                ;;
        esac
    done
    unset OSDC_RESOLVER_READONLY

    export OSDC_ROOT="${ROOT}"
    export OSDC_UPSTREAM="${UPSTREAM}"
    export PYTHONPATH="${UPSTREAM}/tests/smoke:${PYTHONPATH:-}"

    SECONDS=0
    uv run pytest $SMOKE_DIRS \
        -v --tb=short -rs \
        -n auto \
        --cluster-id="$CLUSTER" \
        --rootdir="${UPSTREAM}" \
        && rc=0 || rc=$?
    echo ""
    if (( SECONDS < 60 )); then elapsed="${SECONDS}s"; else elapsed="$((SECONDS / 60))m$((SECONDS % 60))s"; fi
    if [[ "$_AGENT" == "true" ]]; then
        exec 1>&3 2>&4
        if [[ $rc -eq 0 ]]; then echo "OK (${elapsed})"; else cat "$_agent_out"; fi
        rm -f "$_agent_out"
        exit $rc
    fi
    echo "Smoke tests completed in ${elapsed}"
    exit $rc

# Run full integration test against a cluster
integration-test cluster *args:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"

    UPSTREAM="{{UPSTREAM}}"
    ROOT="{{ROOT}}"
    CLUSTER="{{cluster}}"

    # Ensure kubectl is configured for the target cluster
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    just kubeconfig "$CLUSTER"

    export OSDC_ROOT="${ROOT}"
    export OSDC_UPSTREAM="${UPSTREAM}"

    NO_PROXY="${NO_PROXY:-},.eks.amazonaws.com" no_proxy="${no_proxy:-},.eks.amazonaws.com" \
        uv run "${UPSTREAM}/integration-tests/scripts/python/run.py" \
            --cluster-id "$CLUSTER" \
            --clusters-yaml "{{CLUSTERS_YAML}}" \
            --upstream-dir "${UPSTREAM}" \
            --root-dir "${ROOT}" \
            {{args}}

# Run load test against a cluster
load-test cluster *args:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"

    UPSTREAM="{{UPSTREAM}}"
    ROOT="{{ROOT}}"
    CLUSTER="{{cluster}}"

    # Ensure kubectl is configured for the target cluster
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"

    export OSDC_ROOT="${ROOT}"
    export OSDC_UPSTREAM="${UPSTREAM}"

    NO_PROXY="${NO_PROXY:-},.eks.amazonaws.com" no_proxy="${no_proxy:-},.eks.amazonaws.com" \
        uv run "${UPSTREAM}/integration-tests/load-test/scripts/python/load_test_run.py" \
            --cluster-id "$CLUSTER" \
            --clusters-yaml "{{CLUSTERS_YAML}}" \
            --upstream-dir "${UPSTREAM}" \
            --root-dir "${ROOT}" \
            {{args}}

# Run production workload test against a cluster
workload-test cluster *args:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"

    UPSTREAM="{{UPSTREAM}}"
    ROOT="{{ROOT}}"
    CLUSTER="{{cluster}}"

    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    export OSDC_ROOT="${ROOT}"
    export OSDC_UPSTREAM="${UPSTREAM}"

    NO_PROXY="${NO_PROXY:-},.eks.amazonaws.com" no_proxy="${no_proxy:-},.eks.amazonaws.com" \
        uv run "${UPSTREAM}/integration-tests/workload-test/scripts/python/workload_run.py" \
            --cluster-id "$CLUSTER" \
            --clusters-yaml "{{CLUSTERS_YAML}}" \
            --upstream-dir "${UPSTREAM}" \
            {{args}}


# ============================================================================
# ANALYSIS
# ============================================================================

# Analyze runner-to-node packing efficiency
analyze-utilization *args:
    @export OSDC_ROOT="{{ROOT}}"; \
    export OSDC_UPSTREAM="{{UPSTREAM}}"; \
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"; \
    uv run {{SCRIPTS}}/python/analyze_node_utilization.py {{args}}

# Generate ARC runner scale set YAMLs for a cluster (no validation, no apply).
# Mirrors Step 1 of modules/arc-runners/deploy.sh — generation only. Used by
# smoke tests to obtain post-override YAMLs (e.g. proactive_capacity_max
# on staging) without touching the cluster.
generate-arc-runners cluster:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    CLUSTER="{{cluster}}"
    SECONDS=0

    # Agent mode: suppress output on success, show only on failure
    _AGENT="${AGENT_ENVIRONMENT:-false}"
    _agent_out=""
    if [[ "$_AGENT" == "true" ]]; then
        _agent_out=$(mktemp)
        exec 3>&1 4>&2
        exec > "$_agent_out" 2>&1
    fi
    trap '[[ -n "$_agent_out" ]] && rm -f "$_agent_out"' EXIT

    # Resolve module-relative defaults (mirrors deploy.sh).
    MODULE_DIR="{{UPSTREAM}}/modules/arc-runners"
    DEFS_DIR="${ARC_RUNNERS_DEFS_DIR:-$MODULE_DIR/defs}"
    OUTPUT_DIR="${ARC_RUNNERS_OUTPUT_DIR:-$MODULE_DIR/generated}"
    TEMPLATE="${ARC_RUNNERS_TEMPLATE:-$MODULE_DIR/templates/runner.yaml.tpl}"
    MODULE_NAME="${ARC_RUNNERS_MODULE_NAME:-arc-runners}"

    CFG="{{UPSTREAM}}/scripts/cluster-config.py"
    EXPLICIT_TAG=$(uv run "$CFG" "$CLUSTER" arc.runner_image_tag "")
    if [[ -n "$EXPLICIT_TAG" ]]; then
      RUNNER_IMAGE="ghcr.io/actions/actions-runner:$EXPLICIT_TAG"
    else
      RUNNER_IMAGE=$(uv run "$MODULE_DIR/scripts/python/resolve_runner_version.py" "$CLUSTER")
    fi
    export RUNNER_IMAGE

    ARC_RUNNERS_DEFS_DIR="$DEFS_DIR" \
        ARC_RUNNERS_OUTPUT_DIR="$OUTPUT_DIR" \
        ARC_RUNNERS_TEMPLATE="$TEMPLATE" \
        ARC_RUNNERS_MODULE_NAME="$MODULE_NAME" \
        uv run "$MODULE_DIR/scripts/python/generate_runners.py" "$CLUSTER" \
        && rc=0 || rc=$?

    if (( SECONDS < 60 )); then elapsed="${SECONDS}s"; else elapsed="$((SECONDS / 60))m$((SECONDS % 60))s"; fi
    if [[ "$_AGENT" == "true" ]]; then
        exec 1>&3 2>&4
        if [[ $rc -eq 0 ]]; then echo "OK (${elapsed})"; else cat "$_agent_out"; fi
        rm -f "$_agent_out"
        exit $rc
    fi
    if [[ $rc -ne 0 ]]; then
        echo "Generation FAILED. (${elapsed})"
        exit $rc
    fi
    echo "Generated ARC runner configs in ${OUTPUT_DIR} (${elapsed})"

# Monte Carlo cluster simulation for PyTorch CI load
simulate-cluster *args:
    @export OSDC_ROOT="{{ROOT}}"; \
    export OSDC_UPSTREAM="{{UPSTREAM}}"; \
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"; \
    uv run {{SCRIPTS}}/python/simulate_cluster_cli.py \
        --upstream-dir "{{UPSTREAM}}" \
        --consumer-root "{{ROOT}}" \
        {{args}}

# ============================================================================
# LINTING
# ============================================================================

# Lint all code (fast checks — blocks CI, runs all 13 linters in parallel)
lint:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"
    SECONDS=0

    # Agent mode: suppress output on success, show only on failure
    _AGENT="${AGENT_ENVIRONMENT:-false}"
    if [[ "$_AGENT" == "true" ]]; then
        _agent_out=$(mktemp)
        exec 3>&1 4>&2
        exec > "$_agent_out" 2>&1
    fi

    # Helper: collect source directories (upstream + consumer if different)
    _src_dirs() {
        local sub="$1"
        [[ -d "${OSDC_UPSTREAM}/${sub}" ]] && echo "${OSDC_UPSTREAM}/${sub}"
        if [[ "${OSDC_ROOT}" != "${OSDC_UPSTREAM}" ]] && [[ -d "${OSDC_ROOT}/${sub}" ]]; then
            echo "${OSDC_ROOT}/${sub}"
        fi
    }

    WORK=$(mktemp -d)
    trap 'rm -rf "$WORK"; [[ -n "${_agent_out:-}" ]] && rm -f "$_agent_out"' EXIT

    # ── Linter definitions ──────────────────────────────────────────────
    # Each function returns 0 on success, non-zero on failure.
    # stdout/stderr are captured by the parallel runner below.

    lint_tofu_fmt() {
        local rc=0
        for dir in $(_src_dirs modules); do
            tofu fmt -check -recursive "$dir" || rc=1
        done
        return $rc
    }

    lint_shellcheck() {
        local files=()
        for dir in $(_src_dirs base) $(_src_dirs modules) $(_src_dirs scripts); do
            while IFS= read -r -d '' f; do files+=("$f"); done \
                < <(find "$dir" -path '*/.scratch' -prune -o -name '*.sh' -print0 2>/dev/null)
        done
        [[ ${#files[@]} -eq 0 ]] && return 0
        shellcheck "${files[@]}"
    }

    lint_shfmt() {
        local files=()
        for dir in $(_src_dirs base) $(_src_dirs modules) $(_src_dirs scripts); do
            while IFS= read -r -d '' f; do files+=("$f"); done \
                < <(find "$dir" -path '*/.scratch' -prune -o -name '*.sh' -print0 2>/dev/null)
        done
        [[ ${#files[@]} -eq 0 ]] && return 0
        shfmt -d -i 2 -ci -bn "${files[@]}"
    }

    lint_ruff_check() {
        local dirs=()
        for dir in $(_src_dirs base) $(_src_dirs modules) $(_src_dirs scripts); do
            dirs+=("$dir")
        done
        ruff check --config "{{UPSTREAM}}/ruff.toml" "${dirs[@]}"
    }

    lint_ruff_format() {
        local dirs=()
        for dir in $(_src_dirs base) $(_src_dirs modules) $(_src_dirs scripts); do
            dirs+=("$dir")
        done
        ruff format --check --config "{{UPSTREAM}}/ruff.toml" "${dirs[@]}"
    }

    lint_hadolint() {
        local files=()
        for dir in $(_src_dirs base) $(_src_dirs modules); do
            while IFS= read -r -d '' f; do files+=("$f"); done \
                < <(find "$dir" -path '*/.scratch' -prune -o -name 'Dockerfile' -print0 2>/dev/null)
        done
        [[ ${#files[@]} -eq 0 ]] && return 0
        hadolint --config "{{UPSTREAM}}/.hadolint.yaml" "${files[@]}"
    }

    lint_yamllint() {
        local dirs=()
        for dir in $(_src_dirs base/kubernetes) $(_src_dirs base/node-compactor/kubernetes) $(_src_dirs base/helm); do
            dirs+=("$dir")
        done
        for parent in $(_src_dirs modules); do
            for sub in "$parent"/*/kubernetes/ "$parent"/*/helm/; do
                [[ -d "$sub" ]] && dirs+=("$sub")
            done
            for sub in "$parent"/*/defs/; do
                [[ -d "$sub" ]] && dirs+=("$sub")
            done
        done
        [[ ${#dirs[@]} -eq 0 ]] && return 0
        uv run yamllint -c "{{UPSTREAM}}/.yamllint.yaml" "${dirs[@]}" "{{CLUSTERS_YAML}}"
    }

    lint_taplo_check() {
        local files=()
        while IFS= read -r -d '' f; do files+=("$f"); done \
            < <(find "{{UPSTREAM}}" -path '*/.scratch' -prune -o -name '*.toml' -not -path '*/.venv/*' -not -path '*/.terraform/*' -print0)
        if [[ "{{ROOT}}" != "{{UPSTREAM}}" ]]; then
            while IFS= read -r -d '' f; do files+=("$f"); done \
                < <(find "{{ROOT}}" -maxdepth 1 -name '*.toml' -print0 2>/dev/null)
        fi
        [[ ${#files[@]} -eq 0 ]] && return 0
        taplo check "${files[@]}"
    }

    lint_taplo_fmt() {
        local files=()
        while IFS= read -r -d '' f; do files+=("$f"); done \
            < <(find "{{UPSTREAM}}" -path '*/.scratch' -prune -o -name '*.toml' -not -path '*/.venv/*' -not -path '*/.terraform/*' -print0)
        if [[ "{{ROOT}}" != "{{UPSTREAM}}" ]]; then
            while IFS= read -r -d '' f; do files+=("$f"); done \
                < <(find "{{ROOT}}" -maxdepth 1 -name '*.toml' -print0 2>/dev/null)
        fi
        [[ ${#files[@]} -eq 0 ]] && return 0
        taplo fmt --check "${files[@]}"
    }

    lint_kubeconform() {
        local dirs=()
        for dir in $(_src_dirs base/kubernetes) $(_src_dirs base/node-compactor/kubernetes); do
            [[ -f "$dir/kustomization.yaml" ]] && dirs+=("$dir")
        done
        for parent in $(_src_dirs modules); do
            while IFS= read -r -d '' f; do
                dirs+=("$(dirname "$f")")
            done < <(find "$parent" -path '*/.scratch' -prune -o -name 'kustomization.yaml' -print0 2>/dev/null)
        done
        [[ ${#dirs[@]} -eq 0 ]] && return 0
        local LB='{''{'
        local RB='}''}'
        local CRDS_SCHEMA="https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/${LB}.Group${RB}/${LB}.ResourceKind${RB}_${LB}.ResourceAPIVersion${RB}.json"
        local rc=0
        for dir in "${dirs[@]}"; do
            echo "  validating: ${dir#{{UPSTREAM}}/}"
            kubectl kustomize "$dir" 2>/dev/null \
                | kubeconform -strict -summary \
                    -schema-location default \
                    -schema-location "$CRDS_SCHEMA" \
                    -ignore-missing-schemas \
                    -output pretty \
                || rc=1
        done
        return $rc
    }

    lint_kube_linter() {
        local dirs=()
        for dir in $(_src_dirs base/kubernetes) $(_src_dirs base/node-compactor/kubernetes); do
            [[ -f "$dir/kustomization.yaml" ]] && dirs+=("$dir")
        done
        for parent in $(_src_dirs modules); do
            while IFS= read -r -d '' f; do
                dirs+=("$(dirname "$f")")
            done < <(find "$parent" -path '*/.scratch' -prune -o -name 'kustomization.yaml' -print0 2>/dev/null)
        done
        [[ ${#dirs[@]} -eq 0 ]] && return 0
        local rc=0
        for dir in "${dirs[@]}"; do
            echo "  checking: ${dir#{{UPSTREAM}}/}"
            kube-linter lint "$dir" --config "{{UPSTREAM}}/.kube-linter.yaml" || rc=1
        done
        return $rc
    }

    lint_tflint() {
        local roots=()
        for parent in $(_src_dirs modules); do
            while IFS= read -r -d '' f; do
                roots+=("$(dirname "$f")")
            done < <(find "$parent" -path '*/.scratch' -prune -o -name 'main.tf' -not -path '*/modules/*' -print0 2>/dev/null)
        done
        [[ ${#roots[@]} -eq 0 ]] && return 0
        tflint --init --config "{{UPSTREAM}}/.tflint.hcl" 2>/dev/null || true
        local rc=0
        for dir in "${roots[@]}"; do
            echo "  linting: ${dir#{{UPSTREAM}}/}"
            tflint --chdir="$dir" --config "{{UPSTREAM}}/.tflint.hcl" || rc=1
        done
        return $rc
    }

    lint_trivy() {
        local rc=0
        for dir in $(_src_dirs base) $(_src_dirs modules); do
            trivy config --severity HIGH,CRITICAL --exit-code 1 \
                --misconfig-scanners terraform,dockerfile,kubernetes \
                --ignorefile "{{UPSTREAM}}/.trivyignore" \
                --skip-dirs '.scratch' \
                "$dir" || rc=1
        done
        return $rc
    }

    # ── Run all linters in parallel ─────────────────────────────────────
    # Format: "display name:function_name"
    LINTERS=(
        "tofu fmt:lint_tofu_fmt"
        "shellcheck:lint_shellcheck"
        "shfmt:lint_shfmt"
        "ruff check:lint_ruff_check"
        "ruff format:lint_ruff_format"
        "hadolint:lint_hadolint"
        "yamllint:lint_yamllint"
        "taplo check:lint_taplo_check"
        "taplo fmt:lint_taplo_fmt"
        "kubeconform:lint_kubeconform"
        "kube-linter:lint_kube_linter"
        "tflint:lint_tflint"
        "trivy:lint_trivy"
    )

    echo "Running ${#LINTERS[@]} linters in parallel..."

    PIDS=()
    LINTER_NAMES=()
    for entry in "${LINTERS[@]}"; do
        name="${entry%%:*}"
        func="${entry##*:}"
        safe="${name// /_}"
        LINTER_NAMES+=("$name")
        (
            set +e
            "$func"
            echo $? > "$WORK/${safe}.rc"
        ) > "$WORK/${safe}.out" 2>&1 &
        PIDS+=($!)
    done

    # Wait for all linters and collect results
    FAIL_COUNT=0
    FAILED_NAMES=()
    PASSED_NAMES=()
    for i in "${!PIDS[@]}"; do
        wait "${PIDS[$i]}" 2>/dev/null || true
        name="${LINTER_NAMES[$i]}"
        safe="${name// /_}"
        rc=$(cat "$WORK/${safe}.rc" 2>/dev/null || echo 1)
        if [[ "$rc" -eq 0 ]]; then
            PASSED_NAMES+=("$name")
        else
            FAIL_COUNT=$((FAIL_COUNT + 1))
            FAILED_NAMES+=("$name")
        fi
    done
    echo ""

    # Print output only for failed linters
    if [[ $FAIL_COUNT -gt 0 ]]; then
        for name in "${FAILED_NAMES[@]}"; do
            safe="${name// /_}"
            echo "━━━ ${name} FAILED ━━━"
            cat "$WORK/${safe}.out"
            echo ""
        done
    fi

    # Summary
    if (( SECONDS < 60 )); then elapsed="${SECONDS}s"; else elapsed="$((SECONDS / 60))m$((SECONDS % 60))s"; fi
    if [[ "$_AGENT" == "true" ]]; then
        exec 1>&3 2>&4
        if [[ $FAIL_COUNT -eq 0 ]]; then echo "OK (${elapsed})"; else cat "$_agent_out"; fi
        rm -f "$_agent_out"
        [[ $FAIL_COUNT -gt 0 ]] && exit 1 || exit 0
    fi
    if [[ $FAIL_COUNT -gt 0 ]]; then
        UNIQUE_FAILED=$(printf '%s, ' "${FAILED_NAMES[@]}" | sed 's/, $//')
        echo "FAILED linters: ${UNIQUE_FAILED}"
        echo "Lint FAILED: ${#PASSED_NAMES[@]}/${#LINTERS[@]} passed, ${FAIL_COUNT} failed. (${elapsed})"
        exit 1
    fi
    echo "All ${#LINTERS[@]} lint checks passed. (${elapsed})"

# Auto-fix lint issues where possible
lint-fix:
    #!/usr/bin/env bash
    set -euo pipefail
    source "{{UPSTREAM}}/scripts/mise-activate.sh"
    export OSDC_ROOT="{{ROOT}}"
    export OSDC_UPSTREAM="{{UPSTREAM}}"
    export CLUSTERS_YAML="{{CLUSTERS_YAML}}"

    # Helper: collect source directories (upstream + consumer if different)
    _src_dirs() {
        local sub="$1"
        [[ -d "${OSDC_UPSTREAM}/${sub}" ]] && echo "${OSDC_UPSTREAM}/${sub}"
        if [[ "${OSDC_ROOT}" != "${OSDC_UPSTREAM}" ]] && [[ -d "${OSDC_ROOT}/${sub}" ]]; then
            echo "${OSDC_ROOT}/${sub}"
        fi
    }

    echo "━━━ tofu fmt (fix) ━━━"
    for dir in $(_src_dirs modules); do
        tofu fmt -recursive "$dir"
    done

    echo "━━━ shfmt (fix) ━━━"
    SHELL_FILES=()
    for dir in $(_src_dirs base) $(_src_dirs modules) $(_src_dirs scripts); do
        while IFS= read -r -d '' f; do SHELL_FILES+=("$f"); done \
            < <(find "$dir" -path '*/.scratch' -prune -o -name '*.sh' -print0 2>/dev/null)
    done
    if [[ ${#SHELL_FILES[@]} -gt 0 ]]; then
        shfmt -w -i 2 -ci -bn "${SHELL_FILES[@]}"
    fi

    echo "━━━ ruff (fix) ━━━"
    PYTHON_DIRS=()
    for dir in $(_src_dirs base) $(_src_dirs modules) $(_src_dirs scripts); do
        PYTHON_DIRS+=("$dir")
    done
    ruff check --fix --config "{{UPSTREAM}}/ruff.toml" "${PYTHON_DIRS[@]}" || true
    ruff format --config "{{UPSTREAM}}/ruff.toml" "${PYTHON_DIRS[@]}"

    echo "━━━ taplo fmt (fix) ━━━"
    TOML_FILES=()
    while IFS= read -r -d '' f; do TOML_FILES+=("$f"); done \
        < <(find "{{UPSTREAM}}" -path '*/.scratch' -prune -o -name '*.toml' -not -path '*/.venv/*' -not -path '*/.terraform/*' -print0)
    if [[ "{{ROOT}}" != "{{UPSTREAM}}" ]]; then
        while IFS= read -r -d '' f; do TOML_FILES+=("$f"); done \
            < <(find "{{ROOT}}" -maxdepth 1 -name '*.toml' -print0 2>/dev/null)
    fi
    if [[ ${#TOML_FILES[@]} -gt 0 ]]; then
        taplo fmt "${TOML_FILES[@]}"
    fi

    echo ""
    echo "Auto-fix complete. Run 'just lint' to verify."
