{%- if header -%}
{{header}}
{%- if ci -%}
periodics:
{%- else -%}
presubmits:
  {%- if testinfra %}
  kubernetes/test-infra:
  {%- else %}
  kubernetes/kubernetes:
  {%- endif %}
{%- endif %}
{%- endif %}
{%- if not ci %}
{%- set testgrid_dashboards = testgrid_dashboards + ", sig-node-presubmits" %}
{%- endif %}
{%- if "crio" in job_name %}
{%- set testgrid_dashboards = testgrid_dashboards + ", sig-node-cri-o" %}
{%- set runtime = "crio" %}
{%- endif %}
{%- if "containerd" in job_name %}
{%- set testgrid_dashboards = testgrid_dashboards + ", sig-node-containerd" %}
{%- set runtime = "containerd" %}
{%- endif %}
{%- if ci and release_informing %}
{%- set testgrid_dashboards = testgrid_dashboards + ", sig-release-master-informing" %}
{%- set testgrid_alert_email = testgrid_alert_email + ", release-team@kubernetes.io" %}
{%- endif %}
  - name: {{job_name}}
    cluster: {{cluster}}
    {%- if ci %}
    interval: 6h
    {%- else %}
    skip_branches:
    - release-\d+\.\d+  # per-release image
    always_run: false
    {%- if testinfra %}
    run_if_changed: ^jobs/e2e_node/
    {%- elif run_if_changed and presubmit %}
    run_if_changed: {{run_if_changed}}
    {%- endif %}
    optional: true
    {%- if job_type == "node" %}
    skip_report: false
    {%- endif %}
    {%- endif %}
    labels:
      preset-service-account: "true"
      {%- if use_dind %}
      preset-dind-enabled: "true"
      preset-kind-volume-mounts: "true"
      {%- endif %}
      {%- if job_type == "node" %}
      preset-k8s-ssh: "true"
      {%- endif %}
      {%- if ci and job_type != "integration" %}
      prow.k8s.io/refs.org: kubernetes
      prow.k8s.io/refs.repo: kubernetes
      {%- endif %}
    annotations:
      testgrid-dashboards: {{testgrid_dashboards}}
      description: {{description}}
      testgrid-alert-email: {{testgrid_alert_email}}
      {%- if not canary and ( not all_features or presubmit ) %}
      fork-per-release: "true"
      {%- if ci %}
      fork-per-release-periodic-interval: 24h
      fork-per-release-replacements: latest-fast.txt -> latest-{% raw %}{{.Version}}{% endraw %}.txt, https://dl.k8s.io/ci/fast -> https://dl.k8s.io/ci
      {%- endif %}
      {%- endif %}
    decorate: true
    decoration_config:
      timeout: {{timeout}}
    {%- if not ci %}
    path_alias: {% if testinfra %}k8s.io/test-infra{% else %}k8s.io/kubernetes{% endif %}
    {%- endif %}
    extra_refs:
    {%- if need_kubernetes_repo and (ci or testinfra) %}
    - org: kubernetes
      repo: kubernetes
      base_ref: master
      path_alias: k8s.io/kubernetes
      workdir: true
    {%- endif %}
    {%- if need_test_infra_repo and not testinfra %}
    - org: kubernetes
      repo: test-infra
      base_ref: master
      path_alias: k8s.io/test-infra
      auxiliary: true
    {%- endif %}
    {%- if need_containerd_repo %}
    - org: containerd
      repo: containerd
      base_ref: release/2.1
    {%- endif %}
    spec:
      containers:
      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20260807-e16bbc2ef7-master
        command:
        - runner.sh

        {%- if job_type == "node" %}
        args:
        - kubetest2
        - noop
        - --test=node
        - --
        {%- if ci or testinfra %}
        - --use-binaries-from-package
        - --test-package-url=https://dl.k8s.io
        - --test-package-dir=ci/fast
        - --test-package-marker=latest-fast.txt
        {%- else %}
        - --build-binaries
        - --repo-root=.
        {%- endif %}
        - --gcp-zone=us-central1-b
        - --parallelism=1
        - '--label-filter=DRA && Feature: isSubsetOf { {% if all_features %}OffByDefault,{% endif -%} DynamicResourceAllocation } && !Flaky {%- if not ci %} && !Slow {%- endif %}'
        - --timeout={{e2e_node_timeout}}
        - --skip-regex= # Override kubetest2 default in https://github.com/kubernetes-sigs/kubetest2/blob/9f385d26316f5256755bb8fe333970aa5759ec7f/pkg/testers/node/node.go#L92
        - '--test-args={% if all_features -%} --feature-gates=AllAlpha=true,AllBeta=true,EventedPLEG=false --service-feature-gates=AllAlpha=true,AllBeta=true,EventedPLEG=false --runtime-config=api/alpha=true,api/beta=true {% endif -%} --container-runtime-endpoint=unix:///var/run/{{runtime}}/{{runtime}}.sock --container-runtime-process-name=/usr/local/bin/{{runtime}} --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/{{runtime}}.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"{{runtime}}.log\", \"journalctl\": [\"-u\", \"{{runtime}}\"]}"'
        - --image-config-file={{image_config_file}}
        {%- if inject_ssh_public_key %}
        env:
        - name: IGNITION_INJECT_GCE_SSH_PUBLIC_KEY_FILE
          value: "1"
        - name: GOPATH
          value: /go
        - name: KUBE_SSH_USER
          value: core
        {%- endif %}

        {%- elif job_type == "integration" %}
        - /bin/bash
        - -xce
        - |
          # test/e2e_dra is a Go unit test with a dependency on local-up-cluster.sh
          # and the the control plane binaries.
          # We can run it via "make test" to get a JUnit file from gotestsum.
          # The full log output gets enabled to see progress while the test runs
          # and to debug potential cross-test interactions.
          make WHAT="cmd/kube-apiserver cmd/kube-scheduler cmd/kube-controller-manager cmd/kube-proxy cmd/kubelet"
          make test WHAT=./test/e2e_dra FULL_LOG=y KUBE_PRUNE_JUNIT_TESTS=false KUBE_TIMEOUT=-timeout=45m KUBETEST_IN_DOCKER=true CONTROLPLANE_SUDO= KUBELET_SUDO= PROXY_SUDO= CONTAINER_RUNTIME_ENDPOINT=/var/run/docker/containerd/containerd.sock KUBERNETES_SERVER_BIN_DIR="$(pwd)/_output/local/bin/linux/amd64" KUBERNETES_SERVER_CACHE_DIR=/tmp/cache-dir

        {%- elif job_type == "e2e" %}
        args:
        - /bin/bash
        - -xce
        - |
          set -o pipefail
          {%- if ci %}
          # A CI job uses pre-built release artifacts and pulls necessary source files from GitHub.
          revision=$(curl --fail --silent --show-error --location https://dl.k8s.io/ci/fast/latest-fast.txt)
          # Report what was tested.
          echo "{\"revision\":\"$revision\"}" >"${ARTIFACTS}/metadata.json"
          # git hash from e.g. v1.33.0-alpha.1.161+e62ce1c9db2dad
          hash=${revision/*+/}
          kind_yaml_cmd=(curl --fail --silent --show-error --location "https://raw.githubusercontent.com/kubernetes/kubernetes/$hash/test/e2e/dra/kind.yaml")
          kind_node_source="https://dl.k8s.io/ci/fast/$revision/kubernetes-server-linux-amd64.tar.gz"
          {%- else %}
          # A presubmit job uses the checked out and merged source code.
          revision=$(git describe --tags)
          kind_yaml_cmd=(cat test/e2e/dra/kind.yaml)
          kind_node_source=.
          {%- endif %}
          major=$(echo "$revision" | sed -e 's/^v\([0-9]*\).*/\1/')
          minor=$(echo "$revision" | sed -e 's/^v[0-9]*\.\([0-9]*\).*/\1/')
          {%- if all_features %}
          # Which DRA features exist can change over time.
          features=( $( {%- if ci %} curl --fail --silent --show-error --location "https://raw.githubusercontent.com/kubernetes/kubernetes/$hash/pkg/features/kube_features.go" | grep '"DRA' {% else %} grep '"DRA' pkg/features/kube_features.go {%- endif %} | sed 's/.*"\(.*\)"/\1/' ) )
          # These extra features are dependencies for some DRA features but only
          # valid for Kubernetes 1.35+. Additional features here should not
          # affect behavior of tests that don't require them.
          if ((major >= 1 && minor - {{ kubelet_skew }} >= 35)); then
            features+=('GenericWorkload')
          fi
          : "Enabling DRA feature(s): ${features[*]}."
          {%- else %}
          features=( )
          {%- endif %}
          {%- if ci %}
          curl --fail --silent --show-error --location "https://dl.k8s.io/ci/fast/$revision/kubernetes-test-linux-amd64.tar.gz" | tar zxvf -
          ginkgo=kubernetes/test/bin/ginkgo
          e2e_test=kubernetes/test/bin/e2e.test
          {%- else %}
          make WHAT="github.com/onsi/ginkgo/v2/ginkgo k8s.io/kubernetes/test/e2e/e2e.test"
          ginkgo=_output/bin/ginkgo
          e2e_test=_output/bin/e2e.test
          {%- endif %}
          # The latest kind is assumed to work also for older release branches, should this job get forked.
          curl --fail --silent --show-error --location https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" kind
          kind build node-image --image=dra/node:latest "${kind_node_source}"
          GINKGO_E2E_PID=
          trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -TERM "${GINKGO_E2E_PID}"; fi' TERM
          trap 'if [ "${GINKGO_E2E_PID}" ]; then kill -INT "${GINKGO_E2E_PID}"; fi' INT
          # The final kind.yaml the result of getting the original kind.yaml, manipulating it with sed,
          # and adding something at the end.
          (
              ${kind_yaml_cmd[@]}

              # Additional features are not in kind.yaml, but they can be added at the end.
              for feature in ${features[@]}; do echo "  ${feature}: true"; done
          ) >/tmp/kind.yaml

          # Add or extend kubeadmConfigPatches with a ClusterConfiguration which causes etcd to use /tmp
          # (https://github.com/kubernetes-sigs/kind/issues/845#issuecomment-1261248420).
          # kind.yaml may or may not have a `kubeadmConfigPatches`, so we have to be
          # careful.
          if ! grep -q '^kubeadmConfigPatches:$' /tmp/kind.yaml; then
              echo "kubeadmConfigPatches:" >>/tmp/kind.yaml
          fi
          sed -i -e '/^kubeadmConfigPatches:$/a\' -e '  - |\n    kind: ClusterConfiguration\n    etcd:\n      local:\n        dataDir: /tmp/etcd\n' /tmp/kind.yaml

          mkdir -p "${ARTIFACTS}/kind"
          cp /tmp/kind.yaml "${ARTIFACTS}/kind"
          cat /tmp/kind.yaml

          kind create cluster --retain --config /tmp/kind.yaml --image dra/node:latest
          atexit () {
              kind export logs "${ARTIFACTS}/kind"
              kind delete cluster
          }
          trap atexit EXIT
          {%- if kubelet_skew|int > 0 %}

          # Replace the kubelet binary and restart it, as in https://gist.github.com/aojea/2c94034f8e86d08842e5916231eb3fe1
          # and https://github.com/kubernetes/test-infra/blob/9cccc25265537e8dfa556688cf10754622014424/experiment/compatibility-versions/emulated-version-upgrade.sh#L56-L66.
          previous_minor=$((minor - {{kubelet_skew}}))
          if [[ $revision == *alpha.0* ]]; then
              : Treating alpha.0 like previous release because it already gets set on the master branch during code freeze and/or is too similar to the previous release to justify a change in what the jobs test against.
              previous_minor=$((previous_minor - 1))
          fi

          {%- if ci %}
          # Test with the most recent CI build, doesn't even need to be released yet.
          # We want to know if those are broken.
          previous=$(curl --silent -L "https://dl.k8s.io/ci/latest-$major.$previous_minor.txt" )
          curl --silent -L "https://dl.k8s.io/ci/$previous/kubernetes-server-linux-amd64.tar.gz" | tar zxOf - kubernetes/server/bin/kubelet >/tmp/kubelet
          {%- else %}
          # Test with the stable release to avoid breaking presubmits because of unrelated issues in a release candidate.
          # Ask curl to append the HTTP status code after the response body (-w ' %{http_code}').
          # Then parse the output using Bash parameter expansion:
          #   ${response% *}  → everything before the last space (the body)
          #   ${response##* } → everything after the last space (the HTTP code)
          response=$(curl --silent -w ' %{http_code}' -L "https://dl.k8s.io/release/stable-$major.$previous_minor.txt" )
          previous="${response% *}"
          status="${response##* }"
          if [ "$status" == 404 ] ; then
              # if stable doesn't exist - use latest
              response=$(curl --silent -w ' %{http_code}' -L "https://dl.k8s.io/release/latest-$major.$previous_minor.txt" )
              previous="${response% *}"
              status="${response##* }"
          fi
          if [ "$status" -ne 200 ] ; then
              echo "error: unable to get release $major.$previous_minor info, HTTP status: $status, response: $previous"
              exit 1
          fi
          curl --silent -L "https://dl.k8s.io/release/$previous/kubernetes-server-linux-amd64.tar.gz" | tar zxOf - kubernetes/server/bin/kubelet >/tmp/kubelet
          {%- endif %}
          chmod a+rx /tmp/kubelet
          /tmp/kubelet --version
          worker_nodes=$(kind get nodes | grep worker)
          for n in $worker_nodes; do
              docker cp /tmp/kubelet $n:/usr/bin/kubelet
              # DRAExtendedResource has been available since 1.34.
              # We can do version-skew testing against Kubernetes >= 1.36
              # where it is on by default, but only if we enable it
              # explicitly for older kubelets.
              if [[ $previous_minor -ge 34 ]]; then
                  docker exec $n sed -i -e 's/\(KUBELET_EXTRA_ARGS=.*\)/\1 --feature-gates=DRAExtendedResource=true/' /etc/default/kubelet
                  # Verify, intentionally not with --quiet.
                  docker exec $n cat /etc/default/kubelet | grep DRAExtendedResource=true
                  docker exec $n systemctl daemon-reload
              fi
              docker exec $n systemctl restart kubelet
          done

          # Running tests which only cover control plane behavior are not useful
          # in a kubelet version skew job. We can filter them out by including
          # only tests which have the DynamicResourceAllocation feature because
          # only those cover kubelet behavior.
          kubelet_label_filter+=" && Feature: containsAny DynamicResourceAllocation"
          {%- endif %}

          KUBECONFIG=${HOME}/.kube/config ${ginkgo} run --nodes=8 --timeout=24h --silence-skips --force-newlines --no-color {%- if kubelet_skew|int > 0 %} --sem-ver-filter=kubelet=1.$previous_minor {%- endif %} --label-filter="DRA && Feature: isSubsetOf { {% if all_features %}OffByDefault, {% endif -%} DynamicResourceAllocation } {%- if kubelet_skew|int > 0 %}$kubelet_label_filter{%- endif %} && !Flaky {%- if not ci and not allow_slow %} && !Slow {%- endif %}" ${e2e_test} -- -provider=local -report-dir="${ARTIFACTS}" -report-complete-ginkgo -report-complete-junit {%- if all_features %} -logcheck-data-races{%endif%} &
          GINKGO_E2E_PID=$!
          wait "${GINKGO_E2E_PID}"
        {%- if all_features %}
        env:
        # -race gets injected into the build of dynamically linked binaries during "kind build node-image" and "make e2e.test".
        - name: KUBE_RACE
          value: -race
        # Normally control plane components are linked statically, which does not work with -race (needs CGO).
        # We need to override the default for components of interest.
        - name: KUBE_CGO_OVERRIDES
          value: kube-apiserver kube-controller-manager kube-scheduler
        {%- endif %}
        {%- endif %}
        {%- if use_dind %}
        # docker-in-docker needs privileged mode
        securityContext:
          privileged: true
        {%- endif %}
        resources:
          {%- if job_type == "e2e" and all_features %}
          # With DATA RACE detection enabled, more memory is needed.
          limits:
            cpu: 2
            memory: 12Gi
          requests:
            cpu: 2
            memory: 12Gi
          {%- elif ci and job_type == "node" or testinfra %}
          # E2E node jobs just control remote execution and thus
          # need few resources, but only when using release packages.
          limits:
            cpu: 0.5
            memory: 2Gi
          requests:
            cpu: 0.5
            memory: 2Gi
          {%- else %}
          limits:
            cpu: 2
            memory: 6Gi
          requests:
            cpu: 2
            memory: 6Gi
          {%- endif %}

