apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: dcgm-exporter
  namespace: monitoring
  labels:
    app.kubernetes.io/name: dcgm-exporter
    app.kubernetes.io/part-of: osdc-monitoring
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: dcgm-exporter
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app.kubernetes.io/name: dcgm-exporter
        app.kubernetes.io/part-of: osdc-monitoring
    spec:
      # Run ONLY on GPU nodes
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: nvidia.com/gpu.present
                    operator: In
                    values:
                      - "true"
      tolerations:
        # GPU node taints
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
        - key: instance-type
          operator: Exists
          effect: NoSchedule
        - key: node-fleet
          operator: Exists
          effect: NoSchedule
      containers:
        - name: dcgm-exporter
          image: nvcr.io/nvidia/k8s/dcgm-exporter:4.5.2-4.8.1-distroless
          ports:
            - name: metrics
              containerPort: 9400
              protocol: TCP
          args:
            - -f
            - /etc/dcgm-exporter/custom-metrics.csv
            - --collect-interval=60000
          env:
            - name: DCGM_EXPORTER_KUBERNETES
              value: "true"
            - name: DCGM_EXPORTER_LISTEN
              value: ":9400"
            - name: GOMEMLIMIT
              value: "410MiB"
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
            limits:
              cpu: 200m
              memory: 512Mi
          securityContext:
            runAsNonRoot: false
            runAsUser: 0
            capabilities:
              add:
                - SYS_ADMIN
          volumeMounts:
            - name: device-plugins
              mountPath: /var/lib/kubelet/device-plugins
              readOnly: true
            - name: custom-metrics
              mountPath: /etc/dcgm-exporter/custom-metrics.csv
              subPath: custom-metrics.csv
              readOnly: true
      volumes:
        - name: device-plugins
          hostPath:
            path: /var/lib/kubelet/device-plugins
        - name: custom-metrics
          configMap:
            name: dcgm-exporter-metrics
---
apiVersion: v1
kind: Service
metadata:
  name: dcgm-exporter
  namespace: monitoring
  labels:
    app.kubernetes.io/name: dcgm-exporter
    app.kubernetes.io/part-of: osdc-monitoring
spec:
  type: ClusterIP
  clusterIP: None
  selector:
    app.kubernetes.io/name: dcgm-exporter
  ports:
    - name: metrics
      port: 9400
      targetPort: metrics
      protocol: TCP
