# nginx caching reverse proxy for pypiserver.
# Mounted as ConfigMap at /etc/nginx/nginx.conf.
# Runs as non-root with nginx-unprivileged image.
#
# Port layout:
#   nginx :8080 (external, Service-facing)
#   pypiserver :8081 (internal, localhost only)
#
# IMPORTANT: max_size (__NGINX_MAX_CACHE_SIZE__) must stay <= the emptyDir
# sizeLimit for the nginx-cache volume (__NGINX_CACHE_SIZE__ in deployment
# template, default 30Gi).  Both are substituted by deploy.sh at deploy time.

load_module modules/ngx_http_js_module.so;

worker_processes auto;
error_log /dev/stderr warn;
pid /tmp/nginx.pid;

events {
    worker_connections 1024;
}

http {
    # Temp paths — writable directories on emptyDir mounts
    proxy_temp_path /var/cache/nginx/proxy_temp;
    client_body_temp_path /tmp/client_body;
    fastcgi_temp_path /tmp/fastcgi;
    uwsgi_temp_path /tmp/uwsgi;
    scgi_temp_path /tmp/scgi;

    access_log /dev/stdout;

    # Extract date for daily log rotation of fallback logs on EFS.
    # nginx evaluates $time_iso8601 per-request; the map extracts YYYY-MM-DD
    # so each day's fallback requests land in a separate file.
    map $time_iso8601 $log_date {
        default                          'date-unknown';
        '~^(?P<ymd>\d{4}-\d{2}-\d{2})'  $ymd;
    }

    # Cache open log file descriptors.  Re-check the path every 1 minute
    # so the daily rollover at midnight picks up the new filename within
    # ~60 seconds.  Acceptable — the wants-collector processes all files.
    open_log_file_cache max=10 inactive=30s valid=1m min_uses=1;

    # Proxy cache for pypi.org fallback AND local pypiserver responses.
    # keys_zone: 200 MB metadata (~1.6M cached entries)
    # max_size: must be <= emptyDir sizeLimit (see header comment)
    # inactive: purge entries not accessed in 7 days
    proxy_cache_path /var/cache/nginx/pypi
        keys_zone=pypi:200m
        max_size=__NGINX_MAX_CACHE_SIZE__
        inactive=7d
        use_temp_path=off;

    # Re-compress responses for clients.  sub_filter requires uncompressed
    # input (Accept-Encoding suppressed upstream), but clients benefit from
    # gzip on the wire.  Only index page MIME types — not binary downloads.
    gzip on;
    gzip_types application/vnd.pypi.simple.v1+json
               application/vnd.pypi.simple.v1+html;
    gzip_proxied any;
    gzip_vary on;

    # Cache key includes Accept header for PEP 691 content negotiation.
    # pypiserver returns HTML or JSON depending on Accept — different
    # representations of the same URL must be cached separately.
    proxy_cache_key "$request_uri|$http_accept";

    # njs module for merging local + upstream simple index responses.
    js_import merge_indexes from /etc/nginx/merge_indexes.js;

    # Allow njs subrequests to buffer full index responses.  The default
    # 4k is far too small.  Packages like grpcio (~6 MB) and aiohttp
    # (~7 MB) have large indexes.  The root /simple/ listing from
    # pypi.org is ~40 MB.  Set to 100m for generous headroom.
    subrequest_output_buffer_size 100m;

    # Thundering herd protection — only one request to the backend per
    # unique cache key.  Other requests wait for the first one to
    # populate the cache entry.
    proxy_cache_lock on;
    proxy_cache_lock_timeout 10s;
    proxy_cache_lock_age 10s;

    # Stream large wheel files — don't buffer to temp files on disk.
    proxy_max_temp_file_size 0;

    # No upload size limit (future: wheel uploads).
    client_max_body_size 0;

    # Backend: pypiserver gunicorn on localhost.
    # IPv4 loopback is intentional even on IPv6-only EKS: pypiserver's
    # bottle.GunicornServer adapter formats the bind as "%s:%d", which
    # produces malformed "::":8081 if --host :: is passed (gunicorn needs
    # the bracketed form [::]:8081).  The Linux kernel always brings lo up
    # with 127.0.0.1/8 and ::1/128 regardless of pod address family, so
    # 127.0.0.1 is reachable in IPv6-only pods.  pypiserver default
    # interface 0.0.0.0 binds the IPv4 side of lo.
    upstream pypiserver {
        server 127.0.0.1:8081;
        keepalive 16;
    }

    server {
        listen 8080;
        listen [::]:8080;
        server_name _;

        # Root directory for nginx variable access_log.
        # Required: ngx_http_log_script_write() checks root existence
        # before writing any log entry when access_log uses variables.
        # Without this, the default root (/etc/nginx/html) does not exist
        # in nginx-unprivileged, silently dropping all variable-path log entries.
        root /tmp/nginx-root;

        # DNS resolver for proxying to external pypi.org.
        # Use cluster kube-dns with short TTL for reliability.
        # __DNS_RESOLVER__ is substituted by deploy.sh at ConfigMap
        # creation time (reads kube-dns ClusterIP).
        resolver __DNS_RESOLVER__ valid=30s;

        # Rewrite absolute download URLs in pypi.org responses to relative
        # paths so pip fetches packages through this proxy instead of
        # connecting directly to files.pythonhosted.org (which cache-enforcer
        # blocks on runner nodes).
        sub_filter 'https://files.pythonhosted.org' '';
        sub_filter 'https://download.pytorch.org' '';
        sub_filter_once off;
        # text/html is implicit (always filtered by default); listing it
        # again causes a "duplicate MIME type" warning.
        sub_filter_types application/vnd.pypi.simple.v1+json
                         application/vnd.pypi.simple.v1+html;
        sub_filter_last_modified on;

        # nginx-local health check — no backend dependency.
        # NOT loopback-restricted: kubelet HTTP readinessProbe connects from
        # the node IP to the pod IP, not via lo.  The endpoint returns a
        # static 200 with no internal state, so exposure to other in-cluster
        # callers (subject to NetworkPolicy ingress from arc-runners only)
        # is not a concern.
        location = /nginx-health {
            access_log off;
            return 200 "ok\n";
        }

        # Stub status for nginx-prometheus-exporter sidecar.
        # Loopback-only: the exporter sidecar shares the pod network and
        # scrapes via [::1]:8080, so traffic arrives on lo.  Any external
        # pod reaching this endpoint via the Service is denied so connection
        # and request counters aren't exposed cluster-wide.
        location = /stub_status {
            allow 127.0.0.1;
            allow ::1;
            deny all;
            stub_status;
            access_log off;
        }

        # End-to-end health — proxied through to pypiserver (uncached).
        location = /health {
            proxy_pass http://pypiserver;
            proxy_cache off;
            proxy_set_header Host $host;
            proxy_set_header Connection "";
            proxy_http_version 1.1;
        }

        # Index pages: /simple/ and /simple/<package>/
        # njs merges local pypiserver + upstream pypi.org indexes.
        # This resolves the BY/BZ index shadowing problem where pypiserver
        # returns 200 with wrong-variant wheels, preventing fallback to PyPI.
        location ~ ^/simple(/|$) {
            js_content merge_indexes.mergeSimple;
        }

        # Internal: pypiserver index (used by njs subrequest only).
        # Cached with "local:" prefix to avoid collision with upstream.
        location /_internal/local/simple/ {
            internal;
            rewrite ^/_internal/local(.*)$ $1 break;
            proxy_pass http://pypiserver;

            proxy_cache pypi;
            proxy_cache_key "local:$request_uri|$http_accept";
            proxy_cache_valid 200 30m;
            proxy_cache_valid 404 1m;
            proxy_cache_use_stale error timeout updating;
            add_header X-Cache-Status $upstream_cache_status always;

            proxy_connect_timeout 2s;
            proxy_read_timeout 10s;

            proxy_set_header Host $host;
            proxy_set_header Connection "";
            proxy_http_version 1.1;
        }

        # Internal: upstream pypi.org index (used by njs subrequest only).
        # Cached with "upstream:" prefix and longer TTL than local.
        location /_internal/upstream/simple/ {
            internal;
            set $pypi_upstream https://pypi.org;
            rewrite ^/_internal/upstream(.*)$ $1 break;
            proxy_pass $pypi_upstream;

            proxy_cache pypi;
            proxy_cache_key "upstream:$request_uri|$http_accept";
            proxy_cache_valid 200 10m;
            proxy_cache_valid 301 302 1m;
            proxy_cache_valid 404 1m;
            proxy_cache_use_stale error timeout updating
                                  http_500 http_502 http_503;
            add_header X-Cache-Status $upstream_cache_status always;

            proxy_connect_timeout 5s;
            proxy_read_timeout 10s;

            proxy_ssl_server_name on;
            proxy_set_header Host pypi.org;
            proxy_set_header Accept-Encoding "";
            proxy_set_header Connection "";
            proxy_http_version 1.1;
        }

        # PyTorch wheel downloads — proxy to download.pytorch.org.
        # sub_filter rewrites absolute download URLs in index pages to
        # relative /whl/... paths.  This location handles file downloads
        # (.whl, .tar.gz, .zip) with long-term caching (immutable packages).
        # Runners set PIP_EXTRA_INDEX_URL to
        #   http://pypi-cache-{slug}:8080/whl/{cuda}/
        # so all download.pytorch.org traffic is routed through the cache
        # (cache-enforcer does NOT block download.pytorch.org).
        #
        # ORDERING: This regex MUST appear before the generic \.(whl|tar\.gz|zip)$
        # location below.  nginx evaluates regex locations in declaration order
        # and the first match wins.  If the generic block appeared first, it
        # would intercept /whl/... requests and route them to pypiserver
        # instead of download.pytorch.org.
        location ~ ^/whl/.*\.(whl|tar\.gz|zip)$ {
            set $pytorch_upstream https://download.pytorch.org;

            proxy_cache pypi;
            proxy_cache_key $request_uri;
            proxy_cache_valid 200 301 1M;
            proxy_cache_valid 404 1m;
            proxy_cache_use_stale error timeout updating
                                  http_500 http_502 http_503;

            # Allow caching large packages including CUDA wheels (~900 MB).
            # With NVMe-backed cache storage, disk pressure is no longer a concern.
            proxy_max_temp_file_size 2048m;

            proxy_connect_timeout 5s;
            proxy_read_timeout 30s;

            proxy_ssl_server_name on;
            proxy_set_header Host download.pytorch.org;
            proxy_set_header Accept-Encoding "";
            proxy_set_header Connection "";
            proxy_http_version 1.1;

            proxy_redirect https://download.pytorch.org/ /;
            proxy_pass $pytorch_upstream$request_uri;

            add_header X-Cache-Status $upstream_cache_status always;
        }

        # Package downloads from files.pythonhosted.org (hash-based paths).
        # sub_filter rewrites absolute download URLs in pypi.org fallback
        # responses to relative /packages/... paths.  These rewritten URLs
        # use pythonhosted's hash-based structure:
        #   /packages/<2-hex>/<64-hex-sha256>/<filename>.whl
        #
        # This regex matches ONLY hash-based paths (starting with 2 hex
        # chars after /packages/).  Flat paths like /packages/<filename>.whl
        # (generated by pypiserver for local wheelhouse packages) do NOT
        # match — they fall through to the generic \.(whl|tar\.gz|zip)$
        # location below, which routes them to pypiserver.
        #
        # ORDERING: This regex MUST appear before the generic
        # \.(whl|tar\.gz|zip)$ location.  nginx evaluates regex locations
        # in declaration order — if the generic block appeared first, it
        # would intercept /packages/<hash>/... requests and route them to
        # pypiserver (which doesn't serve hash-based paths).
        location ~ ^/packages/[0-9a-f][0-9a-f]/ {
            set $pythonhosted https://files.pythonhosted.org;

            proxy_cache pypi;
            proxy_cache_key $request_uri;
            access_log /data/logs/upstream/fallback.$log_date.log;

            proxy_cache_valid 200 301 1M;
            proxy_cache_valid 404 1m;
            proxy_cache_use_stale error timeout updating
                                  http_500 http_502 http_503;

            # Allow caching large packages including CUDA wheels (~900 MB).
            # With NVMe-backed cache storage, disk pressure is no longer a concern.
            proxy_max_temp_file_size 2048m;

            proxy_connect_timeout 5s;
            proxy_read_timeout 30s;

            proxy_ssl_server_name on;
            proxy_set_header Host files.pythonhosted.org;
            proxy_set_header Accept-Encoding "";
            proxy_set_header Connection "";
            proxy_http_version 1.1;

            proxy_pass $pythonhosted$request_uri;

            add_header X-Cache-Status $upstream_cache_status always;
        }

        # Wheel and tarball files — pypiserver with pypi.org fallback.
        # Handles local wheel downloads (flat /packages/<filename>.whl
        # from pypiserver index pages) and any other .whl/.tar.gz/.zip
        # requests not matched by earlier regex locations.
        # When pypiserver returns 404 (file not in wheelhouse),
        # nginx intercepts and proxies to pypi.org server-side.
        location ~ \.(whl|tar\.gz|zip)$ {
            proxy_pass http://pypiserver;

            # Cache pypiserver wheel downloads.
            # "local:" prefix prevents cache key collision with @pypi_fallback
            # entries for the same URI.  30-day TTL because wheels are immutable.
            # 2048m temp file size allows caching large CUDA wheels (~900MB).
            proxy_cache pypi;
            proxy_cache_key "local:$request_uri";
            proxy_cache_valid 200 30d;
            proxy_max_temp_file_size 2048m;
            add_header X-Cache-Status $upstream_cache_status always;

            # Localhost — fail fast if pypiserver is unresponsive.
            proxy_connect_timeout 2s;
            proxy_read_timeout 10s;

            # Intercept errors from pypiserver and fall through to pypi.org.
            proxy_intercept_errors on;
            error_page 404 500 502 503 = @pypi_fallback;

            proxy_set_header Host $host;
            proxy_set_header Connection "";
            proxy_http_version 1.1;
        }

        # Server-side fallback to upstream PyPI.
        # When pypiserver returns 404 (package not in local wheelhouse),
        # nginx proxies the original request to pypi.org instead of
        # returning the 404 to the client.  This avoids client-side
        # redirects to HTTPS which runner pods cannot follow.
        location @pypi_fallback {
            # Variable assignment forces runtime DNS resolution via resolver.
            set $pypi_upstream https://pypi.org;
            proxy_pass $pypi_upstream$request_uri;
            proxy_cache pypi;
            access_log /data/logs/upstream/fallback.$log_date.log;

            proxy_cache_valid 200 10m;
            proxy_cache_valid 301 302 1m;
            proxy_cache_valid 404 1m;
            proxy_cache_use_stale error timeout updating
                                  http_500 http_502 http_503;

            # Short timeouts — fail fast if pypi.org is unreachable
            # rather than hanging until pip's 15s timeout.
            proxy_connect_timeout 5s;
            proxy_read_timeout 10s;

            # Rewrite pypi.org redirect Location headers to use the local
            # service URL so clients never see HTTPS pypi.org URLs.
            proxy_redirect https://pypi.org/ /;

            proxy_ssl_server_name on;
            proxy_set_header Host pypi.org;
            # Suppress compression so sub_filter can rewrite response bodies.
            proxy_set_header Accept-Encoding "";
            proxy_set_header Connection "";
            proxy_http_version 1.1;

            add_header X-Cache-Status $upstream_cache_status always;
            add_header X-Pypi-Fallback true always;
        }

        # PyTorch simple index — proxy to download.pytorch.org.
        # Serves PEP 503 package listings (e.g. /whl/cu121/torch/).
        # Short cache — index pages change when new versions are released.
        # download.pytorch.org returns 403 for packages it doesn't carry
        # (e.g. /whl/cpu/six/). Tools like uv treat 403 as an auth error
        # and abort resolution. Rewrite 403 → 404 so uv falls through to
        # the default index.
        location /whl {
            set $pytorch_upstream https://download.pytorch.org;

            proxy_cache pypi;
            proxy_cache_key $request_uri;
            proxy_cache_valid 200 10m;
            proxy_cache_valid 301 302 1m;
            proxy_cache_valid 403 404 1m;
            proxy_cache_use_stale error timeout updating
                                  http_500 http_502 http_503;

            # Convert transient upstream 5xx to 404 so uv/pip fall through
            # to the default index instead of failing the whole CI job.
            # proxy_cache_use_stale only helps when a cached entry exists;
            # cold pods (one cache per replica, NVMe hostPath) hit upstream
            # directly and any 5xx propagated to the client.
            proxy_intercept_errors on;
            error_page 403 =404 @pytorch_not_found;
            error_page 500 502 503 504 =404 @pytorch_not_found;

            proxy_connect_timeout 5s;
            proxy_read_timeout 10s;

            proxy_ssl_server_name on;
            proxy_set_header Host download.pytorch.org;
            proxy_set_header Accept-Encoding "";
            proxy_set_header Connection "";
            proxy_http_version 1.1;

            proxy_redirect https://download.pytorch.org/ /;
            proxy_pass $pytorch_upstream$request_uri;

            add_header X-Cache-Status $upstream_cache_status always;
            add_header X-Pytorch-Index true always;
        }

        location @pytorch_not_found {
            default_type text/plain;
            return 404 "Not Found\n";
        }

        # Everything else — pass through uncached.
        location / {
            proxy_pass http://pypiserver;
            proxy_set_header Host $host;
            proxy_set_header Connection "";
            proxy_http_version 1.1;
        }
    }
}
