# Registry Mirror Configuration DaemonSet
# Configures containerd on ALL nodes to use Harbor as a pull-through cache.
# Each upstream registry gets a hosts.toml pointing to Harbor first, with
# the upstream as fallback. If Harbor is down, containerd falls through
# to upstream automatically (connection refused -> next host).
#
# Unlike node-performance-tuning (which targets workload-type: github-runner),
# this runs on ALL nodes including base infrastructure nodes.

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: registry-mirror-config-script
  namespace: kube-system
data:
  configure-mirrors.sh: |
    #!/bin/bash
    set -euo pipefail

    echo "=== Registry Mirror Configuration ==="
    echo "Node: $(hostname 2>/dev/null || echo 'unknown')"
    echo "Date: $(date)"

    # =========================================================================
    # Write /etc/hosts entry: <NODE_IP> harbor
    # =========================================================================
    # Containerd hosts.toml mirror entries below address Harbor as
    # `harbor:30002`. Resolution must be per-node because kube-proxy in
    # IPv6-only mode opens NodePort listeners on `[::]:30002` but does NOT
    # route traffic destined for `::1` to them. The node's own primary IP
    # IS routable. NODE_IP is supplied via the Downward API
    # (`status.hostIP`) on the init container.
    HOSTS_MARKER="# managed-by: osdc-harbor-mirror"
    if [ -n "${NODE_IP:-}" ]; then
      # Idempotent rewrite: drop any prior managed entry, append fresh one.
      # NOTE: /etc/hosts is a kubelet bind-mounted file, NOT a regular file —
      # `sed -i` and any rename(2)-based in-place editor fails with EBUSY
      # because the kernel won't let us swap out a bind-mounted inode.
      # Instead: rewrite into a tmpfile then `cat > /etc/hosts` (which uses
      # O_TRUNC + write, preserving the inode the bind-mount points at).
      HOSTS_TMP=$(mktemp)
      grep -v "${HOSTS_MARKER}" /etc/hosts > "$HOSTS_TMP" || true
      echo "${NODE_IP} harbor ${HOSTS_MARKER}" >> "$HOSTS_TMP"
      cat "$HOSTS_TMP" > /etc/hosts
      rm -f "$HOSTS_TMP"
      echo "Wrote /etc/hosts entry: ${NODE_IP} harbor"
    else
      echo "ERROR: NODE_IP env var not set; cannot configure /etc/hosts" >&2
      exit 1
    fi

    # =========================================================================
    # Check if already configured
    # =========================================================================
    MARKER_FILE="/var/lib/registry-mirror-config/.configured"
    HARBOR_PORT="${HARBOR_PORT:-30002}"

    if [ -f "$MARKER_FILE" ]; then
      echo ""
      echo "Node already configured (marker file exists)"
      echo "Verifying current configuration..."

      all_checks_pass=true
      for registry in docker.io ghcr.io public.ecr.aws nvcr.io registry.k8s.io quay.io "harbor:${HARBOR_PORT}"; do
        if [ ! -f "/etc/containerd/certs.d/${registry}/hosts.toml" ]; then
          echo "  Missing hosts.toml for ${registry} - will reconfigure"
          all_checks_pass=false
          break
        fi
      done

      if [ "$all_checks_pass" = true ]; then
        echo "All mirror configs present - skipping reconfiguration"
        exit 0
      else
        echo "Configuration drift detected - proceeding with reconfiguration"
        rm -f "$MARKER_FILE"
      fi
    fi

    # =========================================================================
    # Registry-to-Harbor project mapping
    # =========================================================================
    # Registry            | Harbor Project
    # --------------------|----------------
    # docker.io           | dockerhub-cache
    # ghcr.io             | ghcr-cache
    # public.ecr.aws      | ecr-public-cache
    # nvcr.io             | nvcr-cache
    # registry.k8s.io     | k8s-cache
    # quay.io             | quay-cache

    echo ""
    echo "[1/6] Configuring docker.io mirror..."
    mkdir -p /etc/containerd/certs.d/docker.io
    cat > /etc/containerd/certs.d/docker.io/hosts.toml <<TOML
    server = "https://docker.io"

    [host."http://harbor:${HARBOR_PORT}/v2/dockerhub-cache"]
      capabilities = ["pull", "resolve"]
      skip_verify = true
      override_path = true

    [host."https://docker.io"]
      capabilities = ["pull", "resolve"]
    TOML

    echo "[2/6] Configuring ghcr.io mirror..."
    mkdir -p /etc/containerd/certs.d/ghcr.io
    cat > /etc/containerd/certs.d/ghcr.io/hosts.toml <<TOML
    server = "https://ghcr.io"

    [host."http://harbor:${HARBOR_PORT}/v2/ghcr-cache"]
      capabilities = ["pull", "resolve"]
      skip_verify = true
      override_path = true

    [host."https://ghcr.io"]
      capabilities = ["pull", "resolve"]
    TOML

    echo "[3/6] Configuring public.ecr.aws mirror..."
    mkdir -p /etc/containerd/certs.d/public.ecr.aws
    cat > /etc/containerd/certs.d/public.ecr.aws/hosts.toml <<TOML
    server = "https://public.ecr.aws"

    [host."http://harbor:${HARBOR_PORT}/v2/ecr-public-cache"]
      capabilities = ["pull", "resolve"]
      skip_verify = true
      override_path = true

    [host."https://public.ecr.aws"]
      capabilities = ["pull", "resolve"]
    TOML

    echo "[4/6] Configuring nvcr.io mirror..."
    mkdir -p /etc/containerd/certs.d/nvcr.io
    cat > /etc/containerd/certs.d/nvcr.io/hosts.toml <<TOML
    server = "https://nvcr.io"

    [host."http://harbor:${HARBOR_PORT}/v2/nvcr-cache"]
      capabilities = ["pull", "resolve"]
      skip_verify = true
      override_path = true

    [host."https://nvcr.io"]
      capabilities = ["pull", "resolve"]
    TOML

    echo "[5/6] Configuring registry.k8s.io mirror..."
    mkdir -p /etc/containerd/certs.d/registry.k8s.io
    cat > /etc/containerd/certs.d/registry.k8s.io/hosts.toml <<TOML
    server = "https://registry.k8s.io"

    [host."http://harbor:${HARBOR_PORT}/v2/k8s-cache"]
      capabilities = ["pull", "resolve"]
      skip_verify = true
      override_path = true

    [host."https://registry.k8s.io"]
      capabilities = ["pull", "resolve"]
    TOML

    echo "[6/6] Configuring quay.io mirror..."
    mkdir -p /etc/containerd/certs.d/quay.io
    cat > /etc/containerd/certs.d/quay.io/hosts.toml <<TOML
    server = "https://quay.io"

    [host."http://harbor:${HARBOR_PORT}/v2/quay-cache"]
      capabilities = ["pull", "resolve"]
      skip_verify = true
      override_path = true

    [host."https://quay.io"]
      capabilities = ["pull", "resolve"]
    TOML

    # =========================================================================
    # Direct Harbor registry (for images pushed to Harbor as `harbor:30002/...`)
    # =========================================================================
    # Workloads like node-compactor / image-cache-janitor / runner-base pull
    # images by their Harbor-native reference (`harbor:30002/osdc/...`). Without
    # a hosts.toml for the `harbor:<port>` hostname, containerd defaults to
    # HTTPS, but Harbor's NodePort only serves plaintext HTTP — pull fails with
    # `http: server gave HTTP response to HTTPS client`. Spell it out here so
    # containerd uses plain HTTP for this registry.
    echo "[harbor] Configuring direct harbor:${HARBOR_PORT} registry (plain HTTP)..."
    mkdir -p "/etc/containerd/certs.d/harbor:${HARBOR_PORT}"
    cat > "/etc/containerd/certs.d/harbor:${HARBOR_PORT}/hosts.toml" <<TOML
    server = "http://harbor:${HARBOR_PORT}"

    [host."http://harbor:${HARBOR_PORT}"]
      capabilities = ["pull", "resolve"]
      skip_verify = true
    TOML

    # =========================================================================
    # Summary
    # =========================================================================
    echo ""
    echo "=== Registry Mirror Configuration Complete ==="
    echo "Harbor endpoint: http://harbor:${HARBOR_PORT}"
    echo "Configured mirrors for: docker.io, ghcr.io, public.ecr.aws, nvcr.io, registry.k8s.io, quay.io"
    echo "Configured direct registry: harbor:${HARBOR_PORT}"
    echo ""
    echo "Note: containerd 1.6+ dynamically watches certs.d/ - no restart needed"

    # Create marker file
    mkdir -p /var/lib/registry-mirror-config || true
    echo "$(date -Iseconds 2>/dev/null || date)" > "$MARKER_FILE" || true

    # Remove the per-DS startup taint so workload pods can schedule on this node.
    # Declared in modules/nodepools/scripts/python/generate_nodepools.py
    # (MODULE_STARTUP_TAINTS registry — added in a follow-up PR). Idempotent:
    # silent no-op when the taint isn't present.
    python3 /scripts/taint-remover/taint_remover.py node-init.osdc.io/registry-mirror

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: registry-mirror-config
  namespace: kube-system
  labels:
    app: registry-mirror-config
    app.kubernetes.io/name: registry-mirror-config
    app.kubernetes.io/component: node-configuration
spec:
  selector:
    matchLabels:
      app: registry-mirror-config

  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      # This DaemonSet only runs an init container (writes /etc/hosts +
      # containerd certs.d) then sleeps — it is config-only, not part of
      # the request path. A `maxUnavailable: 1` setting causes the entire
      # rollout to gate on any single stuck Terminating pod (e.g. on a
      # Karpenter-disrupted node where kubelet can't ack the termination),
      # leaving the rest of the fleet without the new config indefinitely.
      # 50% keeps half the fleet on the old config while a rollout proceeds
      # so a bad script revision can still be caught before it lands on
      # every node, but a single unreachable node can no longer block it.
      maxUnavailable: "50%"

  template:
    metadata:
      labels:
        app: registry-mirror-config

    spec:
      hostNetwork: true
      hostPID: true

      priorityClassName: system-node-critical

      # Run on ALL nodes (base infra + Karpenter-managed)
      tolerations:
        - operator: Exists

      # NO nodeSelector - must run on every node in the cluster

      serviceAccountName: registry-mirror-config

      initContainers:
        - name: configure-mirrors
          image: public.ecr.aws/amazonlinux/amazonlinux:2023
          command: ["/bin/bash", "-c"]
          args:
            - /scripts/configure-mirrors.sh

          env:
            - name: HARBOR_PORT
              value: "30002"
            - name: NODE_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName

          securityContext:
            privileged: true

          volumeMounts:
            - name: scripts
              mountPath: /scripts
              readOnly: true
            - name: taint-remover-lib
              mountPath: /scripts/taint-remover
              readOnly: true
            - name: etc-containerd
              mountPath: /etc/containerd
            - name: var-lib-registry-mirror-config
              mountPath: /var/lib/registry-mirror-config
            - name: etc-hosts
              mountPath: /etc/hosts

      containers:
        - name: sleep
          image: public.ecr.aws/docker/library/alpine:3.19
          command: ["/bin/sh", "-c", "echo 'Registry mirror config complete. Sleeping...'; sleep infinity"]

          resources:
            requests:
              cpu: 10m
              memory: 32Mi
            limits:
              cpu: 50m
              memory: 64Mi

      volumes:
        - name: scripts
          configMap:
            name: registry-mirror-config-script
            defaultMode: 0755

        - name: taint-remover-lib
          configMap:
            name: node-taint-remover-lib

        - name: etc-containerd
          hostPath:
            path: /etc/containerd
            type: DirectoryOrCreate

        - name: var-lib-registry-mirror-config
          hostPath:
            path: /var/lib/registry-mirror-config
            type: DirectoryOrCreate

        - name: etc-hosts
          hostPath:
            path: /etc/hosts
            type: File

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: registry-mirror-config
  namespace: kube-system
  labels:
    app: registry-mirror-config
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: registry-mirror-config-taint-remover
rules:
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: registry-mirror-config-taint-remover
subjects:
  - kind: ServiceAccount
    name: registry-mirror-config
    namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: registry-mirror-config-taint-remover
