// Copyright (c) 2023 Volvo Car Corporation
// SPDX-License-Identifier: Apache-2.0

// Code generated by lingon. EDIT AS MUCH AS YOU LIKE.

package vmk8s

import (
	vmo "github.com/VictoriaMetrics/operator/api/operator/v1"
	"github.com/golingon/lingon/pkg/kube"
	ku "github.com/golingon/lingon/pkg/kubeutil"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type K8SRules struct {
	kube.App

	K8SRecordingRules             *vmo.VMRule
	K8SGeneralAlertRules          *vmo.VMRule
	PromGeneralRules              *vmo.VMRule
	NodeRecordingRules            *vmo.VMRule
	KubeletRecordingRules         *vmo.VMRule
	KubernetesAppsAlertRules      *vmo.VMRule
	KubernetesResourcesAlertRules *vmo.VMRule
	KubernetesStorageAlertRules   *vmo.VMRule
	KubeletAlertRules             *vmo.VMRule
	KubernetesSystemAlertRules    *vmo.VMRule
	NodeNetworkAlertRules         *vmo.VMRule
	K8SNodeRules                  *vmo.VMRule
}

func NewK8SRules() *K8SRules {
	return &K8SRules{
		K8SRecordingRules:             K8SRecordingRules,
		K8SGeneralAlertRules:          K8SGeneralAlertRules,
		PromGeneralRules:              PromGeneralRules,
		NodeRecordingRules:            NodeRecordingRules,
		KubeletRecordingRules:         KubeletRecordingRules,
		KubernetesAppsAlertRules:      KubernetesAppsAlertRules,
		KubernetesResourcesAlertRules: KubernetesResourcesAlertRules,
		KubernetesStorageAlertRules:   KubernetesStorageAlertRules,
		KubeletAlertRules:             KubeletAlertRules,
		KubernetesSystemAlertRules:    KubernetesSystemAlertRules,
		NodeNetworkAlertRules:         NodeNetworkAlertRules,
		K8SNodeRules:                  K8SNodeRules,
	}
}

var RulesLabels = map[string]string{
	"app":                Single.Name,
	ku.AppLabelName:      Single.Name,
	ku.AppLabelInstance:  Single.Instance,
	ku.AppLabelComponent: "rules",
	ku.AppLabelPartOf:    Single.PartOf,
	ku.AppLabelVersion:   Single.Version,
	ku.AppLabelManagedBy: "lingon",
}

var K8SGeneralAlertRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-general.rules",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "general.rules",
				Rules: []vmo.Rule{
					{
						Alert: "TargetDown",
						Annotations: map[string]string{
							"description": `{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service }} targets in {{ $labels.namespace }} namespace are down.`,
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/general/targetdown",
							"summary":     "One or more targets are unreachable.",
						},
						Expr:   "100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10",
						For:    "10m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "Watchdog",
						Annotations: map[string]string{
							"description": `This is an alert meant to ensure that the entire alerting pipeline is functional. This alert is always firing, therefore it should always be firing in Alertmanager and always fire against a receiver. There are integrations with various notification mechanisms that send a notification when this alert is not firing. For example the "DeadMansSnitch" integration in PagerDuty. `,
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/general/watchdog",
							"summary":     "An alert that should always be firing to certify that Alertmanager is working properly.",
						},
						Expr:   "vector(1)",
						Labels: map[string]string{"severity": "none"},
					}, {
						Alert: "InfoInhibitor",
						Annotations: map[string]string{
							"description": `This is an alert that is used to inhibit info alerts. By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with other alerts. This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a severity of 'warning' or 'critical' starts firing on the same namespace. This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". `,
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor",
							"summary":     "Info-level alert inhibition.",
						},
						Expr:   `ALERTS{severity = "info"} == 1 unless on(namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1`,
						Labels: map[string]string{"severity": "none"},
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var K8SRecordingRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-k8s.rules",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "k8s.rules",
				Rules: []vmo.Rule{
					{
						Expr: `
sum by (cluster, namespace, pod, container) (
  irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m])
) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
  1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""})
)
`,
						Record: "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate",
					}, {
						Expr: `
container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
* on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1,
  max by(cluster, namespace, pod, node) (kube_pod_info{node!=""})
)
`,
						Record: "node_namespace_pod_container:container_memory_working_set_bytes",
					}, {
						Expr: `
container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
* on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1,
  max by(cluster, namespace, pod, node) (kube_pod_info{node!=""})
)
`,
						Record: "node_namespace_pod_container:container_memory_rss",
					}, {
						Expr: `
container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
* on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1,
  max by(cluster, namespace, pod, node) (kube_pod_info{node!=""})
)
`,
						Record: "node_namespace_pod_container:container_memory_cache",
					}, {
						Expr: `
container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
* on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1,
  max by(cluster, namespace, pod, node) (kube_pod_info{node!=""})
)
`,
						Record: "node_namespace_pod_container:container_memory_swap",
					}, {
						Expr: `
kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}  * on (namespace, pod, cluster)
group_left() max by (namespace, pod, cluster) (
  (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
)
`,
						Record: "cluster:namespace:pod_memory:active:kube_pod_container_resource_requests",
					}, {
						Expr: `
sum by (namespace, cluster) (
    sum by (namespace, pod, cluster) (
        max by (namespace, pod, container, cluster) (
          kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}
        ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
          kube_pod_status_phase{phase=~"Pending|Running"} == 1
        )
    )
)
`,
						Record: "namespace_memory:kube_pod_container_resource_requests:sum",
					}, {
						Expr: `
kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}  * on (namespace, pod, cluster)
group_left() max by (namespace, pod, cluster) (
  (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
)
`,
						Record: "cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests",
					}, {
						Expr: `
sum by (namespace, cluster) (
    sum by (namespace, pod, cluster) (
        max by (namespace, pod, container, cluster) (
          kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}
        ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
          kube_pod_status_phase{phase=~"Pending|Running"} == 1
        )
    )
)
`,
						Record: "namespace_cpu:kube_pod_container_resource_requests:sum",
					}, {
						Expr: `
kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}  * on (namespace, pod, cluster)
group_left() max by (namespace, pod, cluster) (
  (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
)
`,
						Record: "cluster:namespace:pod_memory:active:kube_pod_container_resource_limits",
					}, {
						Expr: `
sum by (namespace, cluster) (
    sum by (namespace, pod, cluster) (
        max by (namespace, pod, container, cluster) (
          kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}
        ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
          kube_pod_status_phase{phase=~"Pending|Running"} == 1
        )
    )
)
`,
						Record: "namespace_memory:kube_pod_container_resource_limits:sum",
					}, {
						Expr: `
kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}  * on (namespace, pod, cluster)
group_left() max by (namespace, pod, cluster) (
 (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
 )
`,
						Record: "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits",
					}, {
						Expr: `
sum by (namespace, cluster) (
    sum by (namespace, pod, cluster) (
        max by (namespace, pod, container, cluster) (
          kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}
        ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
          kube_pod_status_phase{phase=~"Pending|Running"} == 1
        )
    )
)
`,
						Record: "namespace_cpu:kube_pod_container_resource_limits:sum",
					}, {
						Expr: `
max by (cluster, namespace, workload, pod) (
  label_replace(
    label_replace(
      kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"},
      "replicaset", "$1", "owner_name", "(.*)"
    ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (
      1, max by (replicaset, namespace, owner_name) (
        kube_replicaset_owner{job="kube-state-metrics"}
      )
    ),
    "workload", "$1", "owner_name", "(.*)"
  )
)
`,
						Labels: map[string]string{"workload_type": "deployment"},
						Record: "namespace_workload_pod:kube_pod_owner:relabel",
					}, {
						Expr: `
max by (cluster, namespace, workload, pod) (
  label_replace(
    kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"},
    "workload", "$1", "owner_name", "(.*)"
  )
)
`,
						Labels: map[string]string{"workload_type": "daemonset"},
						Record: "namespace_workload_pod:kube_pod_owner:relabel",
					}, {
						Expr: `
max by (cluster, namespace, workload, pod) (
  label_replace(
    kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"},
    "workload", "$1", "owner_name", "(.*)"
  )
)
`,
						Labels: map[string]string{"workload_type": "statefulset"},
						Record: "namespace_workload_pod:kube_pod_owner:relabel",
					}, {
						Expr: `
max by (cluster, namespace, workload, pod) (
  label_replace(
    kube_pod_owner{job="kube-state-metrics", owner_kind="Job"},
    "workload", "$1", "owner_name", "(.*)"
  )
)
`,
						Labels: map[string]string{"workload_type": "job"},
						Record: "namespace_workload_pod:kube_pod_owner:relabel",
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var PromGeneralRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-kube-prometheus-general.rules",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "kube-prometheus-general.rules",
				Rules: []vmo.Rule{
					{
						Expr:   "count without(instance, pod, node) (up == 1)",
						Record: "count:up1",
					}, {
						Expr:   "count without(instance, pod, node) (up == 0)",
						Record: "count:up0",
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var NodeRecordingRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-kube-prometheus-node-recording",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "kube-prometheus-node-recording.rules",
				Rules: []vmo.Rule{
					{
						Expr:   `sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance)`,
						Record: "instance:node_cpu:rate:sum",
					}, {
						Expr:   "sum(rate(node_network_receive_bytes_total[3m])) BY (instance)",
						Record: "instance:node_network_receive_bytes:rate:sum",
					}, {
						Expr:   "sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)",
						Record: "instance:node_network_transmit_bytes:rate:sum",
					}, {
						Expr:   `sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance)`,
						Record: "instance:node_cpu:ratio",
					}, {
						Expr:   `sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m]))`,
						Record: "cluster:node_cpu:sum_rate5m",
					}, {
						Expr:   "cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))",
						Record: "cluster:node_cpu:ratio",
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var KubeletRecordingRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-kubelet.rules",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "kubelet.rules",
				Rules: []vmo.Rule{
					{
						Expr:   `histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})`,
						Labels: map[string]string{"quantile": "0.99"},
						Record: "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile",
					}, {
						Expr:   `histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})`,
						Labels: map[string]string{"quantile": "0.9"},
						Record: "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile",
					}, {
						Expr:   `histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})`,
						Labels: map[string]string{"quantile": "0.5"},
						Record: "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile",
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var KubernetesAppsAlertRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-kubernetes-apps",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "kubernetes-apps",
				Rules: []vmo.Rule{
					{
						Alert: "KubePodCrashLooping",
						Annotations: map[string]string{
							"description": `Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: "CrashLoopBackOff").`,
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping",
							"summary":     "Pod is crash looping.",
						},
						Expr:   `max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="kube-state-metrics", namespace=~".*"}[5m]) >= 1`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubePodNotReady",
						Annotations: map[string]string{
							"description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready",
							"summary":     "Pod has been in a non-ready state for more than 15 minutes.",
						},
						Expr: `
sum by (namespace, pod, cluster) (
  max by(namespace, pod, cluster) (
    kube_pod_status_phase{job="kube-state-metrics", namespace=~".*", phase=~"Pending|Unknown|Failed"}
  ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) (
    1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"})
  )
) > 0
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeDeploymentGenerationMismatch",
						Annotations: map[string]string{
							"description": "Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch",
							"summary":     "Deployment generation mismatch due to possible roll-back",
						},
						Expr: `
kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
  !=
kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~".*"}
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeDeploymentReplicasMismatch",
						Annotations: map[string]string{
							"description": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch",
							"summary":     "Deployment has not matched the expected number of replicas.",
						},
						Expr: `
(
  kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~".*"}
    >
  kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~".*"}
) and (
  changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
    ==
  0
)
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeStatefulSetReplicasMismatch",
						Annotations: map[string]string{
							"description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch",
							"summary":     "Deployment has not matched the expected number of replicas.",
						},
						Expr: `
(
  kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~".*"}
    !=
  kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~".*"}
) and (
  changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
    ==
  0
)
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeStatefulSetGenerationMismatch",
						Annotations: map[string]string{
							"description": "StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch",
							"summary":     "StatefulSet generation mismatch due to possible roll-back",
						},
						Expr: `
kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
  !=
kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~".*"}
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeStatefulSetUpdateNotRolledOut",
						Annotations: map[string]string{
							"description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout",
							"summary":     "StatefulSet update has not been rolled out.",
						},
						Expr: `
(
  max without (revision) (
    kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~".*"}
      unless
    kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~".*"}
  )
    *
  (
    kube_statefulset_replicas{job="kube-state-metrics", namespace=~".*"}
      !=
    kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}
  )
)  and (
  changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[5m])
    ==
  0
)
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeDaemonSetRolloutStuck",
						Annotations: map[string]string{
							"description": "DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck",
							"summary":     "DaemonSet rollout is stuck.",
						},
						Expr: `
(
  (
    kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"}
     !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
  ) or (
    kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"}
     !=
    0
  ) or (
    kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}
     !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
  ) or (
    kube_daemonset_status_number_available{job="kube-state-metrics", namespace=~".*"}
     !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
  )
) and (
  changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}[5m])
    ==
  0
)
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeContainerWaiting",
						Annotations: map[string]string{
							"description": "pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting",
							"summary":     "Pod container waiting longer than 1 hour",
						},
						Expr:   `sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", namespace=~".*"}) > 0`,
						For:    "1h",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeDaemonSetNotScheduled",
						Annotations: map[string]string{
							"description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled",
							"summary":     "DaemonSet pods are not scheduled.",
						},
						Expr: `
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
  -
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"} > 0
`,
						For:    "10m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeDaemonSetMisScheduled",
						Annotations: map[string]string{
							"description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled",
							"summary":     "DaemonSet pods are misscheduled.",
						},
						Expr:   `kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"} > 0`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeJobNotCompleted",
						Annotations: map[string]string{
							"description": `Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "43200" | humanizeDuration }} to complete.`,
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted",
							"summary":     "Job did not complete in time",
						},
						Expr: `
time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics", namespace=~".*"}
  and
kube_job_status_active{job="kube-state-metrics", namespace=~".*"} > 0) > 43200
`,
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeJobFailed",
						Annotations: map[string]string{
							"description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed",
							"summary":     "Job failed to complete.",
						},
						Expr:   `kube_job_failed{job="kube-state-metrics", namespace=~".*"}  > 0`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeHpaReplicasMismatch",
						Annotations: map[string]string{
							"description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has not matched the desired number of replicas for longer than 15 minutes.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch",
							"summary":     "HPA has not matched desired number of replicas.",
						},
						Expr: `
(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics", namespace=~".*"}
  !=
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"})
  and
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
  >
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics", namespace=~".*"})
  and
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
  <
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"})
  and
changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}[15m]) == 0
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeHpaMaxedOut",
						Annotations: map[string]string{
							"description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has been running at max replicas for longer than 15 minutes.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout",
							"summary":     "HPA is running at max replicas",
						},
						Expr: `
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
  ==
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"}
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var KubernetesResourcesAlertRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-kubernetes-resources",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "kubernetes-resources",
				Rules: []vmo.Rule{
					{
						Alert: "KubeCPUOvercommit",
						Annotations: map[string]string{
							"description": "Cluster has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit",
							"summary":     "Cluster has overcommitted CPU resource requests.",
						},
						Expr: `
sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0
and
(sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0
`,
						For:    "10m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeMemoryOvercommit",
						Annotations: map[string]string{
							"description": "Cluster has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit",
							"summary":     "Cluster has overcommitted memory resource requests.",
						},
						Expr: `
sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0
and
(sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0
`,
						For:    "10m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeCPUQuotaOvercommit",
						Annotations: map[string]string{
							"description": "Cluster has overcommitted CPU resource requests for Namespaces.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit",
							"summary":     "Cluster has overcommitted CPU resource requests.",
						},
						Expr: `
sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"}))
  /
sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"})
  > 1.5
`,
						For:    "5m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeMemoryQuotaOvercommit",
						Annotations: map[string]string{
							"description": "Cluster has overcommitted memory resource requests for Namespaces.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit",
							"summary":     "Cluster has overcommitted memory resource requests.",
						},
						Expr: `
sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"}))
  /
sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"})
  > 1.5
`,
						For:    "5m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeQuotaAlmostFull",
						Annotations: map[string]string{
							"description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull",
							"summary":     "Namespace quota is going to be full.",
						},
						Expr: `
kube_resourcequota{job="kube-state-metrics", type="used"}
  / ignoring(instance, job, type)
(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
  > 0.9 < 1
`,
						For:    "15m",
						Labels: map[string]string{"severity": "info"},
					}, {
						Alert: "KubeQuotaFullyUsed",
						Annotations: map[string]string{
							"description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused",
							"summary":     "Namespace quota is fully used.",
						},
						Expr: `
kube_resourcequota{job="kube-state-metrics", type="used"}
  / ignoring(instance, job, type)
(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
  == 1
`,
						For:    "15m",
						Labels: map[string]string{"severity": "info"},
					}, {
						Alert: "KubeQuotaExceeded",
						Annotations: map[string]string{
							"description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded",
							"summary":     "Namespace quota has exceeded the limits.",
						},
						Expr: `
kube_resourcequota{job="kube-state-metrics", type="used"}
  / ignoring(instance, job, type)
(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
  > 1
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "CPUThrottlingHigh",
						Annotations: map[string]string{
							"description": "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh",
							"summary":     "Processes experience elevated CPU throttling.",
						},
						Expr: `
sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
  /
sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
  > ( 25 / 100 )
`,
						For:    "15m",
						Labels: map[string]string{"severity": "info"},
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var KubernetesStorageAlertRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-kubernetes-storage",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "kubernetes-storage",
				Rules: []vmo.Rule{
					{
						Alert: "KubePersistentVolumeFillingUp",
						Annotations: map[string]string{
							"description": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup",
							"summary":     "PersistentVolume is filling up.",
						},
						Expr: `
(
  kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
    /
  kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
) < 0.03
and
kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
`,
						For:    "1m",
						Labels: map[string]string{"severity": "critical"},
					}, {
						Alert: "KubePersistentVolumeFillingUp",
						Annotations: map[string]string{
							"description": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup",
							"summary":     "PersistentVolume is filling up.",
						},
						Expr: `
(
  kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
    /
  kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
) < 0.15
and
kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
and
predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
`,
						For:    "1h",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubePersistentVolumeInodesFillingUp",
						Annotations: map[string]string{
							"description": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup",
							"summary":     "PersistentVolumeInodes are filling up.",
						},
						Expr: `
(
  kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}
    /
  kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
) < 0.03
and
kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
`,
						For:    "1m",
						Labels: map[string]string{"severity": "critical"},
					}, {
						Alert: "KubePersistentVolumeInodesFillingUp",
						Annotations: map[string]string{
							"description": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup",
							"summary":     "PersistentVolumeInodes are filling up.",
						},
						Expr: `
(
  kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}
    /
  kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
) < 0.15
and
kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
and
predict_linear(kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
`,
						For:    "1h",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubePersistentVolumeErrors",
						Annotations: map[string]string{
							"description": "The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors",
							"summary":     "PersistentVolume is having issues with provisioning.",
						},
						Expr:   `kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0`,
						For:    "5m",
						Labels: map[string]string{"severity": "critical"},
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var KubeletAlertRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-kubelet",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "kubernetes-system-kubelet",
				Rules: []vmo.Rule{
					{
						Alert: "KubeNodeNotReady",
						Annotations: map[string]string{
							"description": "{{ $labels.node }} has been unready for more than 15 minutes.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready",
							"summary":     "Node is not ready.",
						},
						Expr:   `kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeNodeUnreachable",
						Annotations: map[string]string{
							"description": "{{ $labels.node }} is unreachable and some workloads may be rescheduled.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeunreachable",
							"summary":     "Node is unreachable.",
						},
						Expr:   `(kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeletTooManyPods",
						Annotations: map[string]string{
							"description": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods",
							"summary":     "Kubelet is running at capacity.",
						},
						Expr: `
count by(cluster, node) (
  (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
)
/
max by(cluster, node) (
  kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1
) > 0.95
`,
						For:    "15m",
						Labels: map[string]string{"severity": "info"},
					}, {
						Alert: "KubeNodeReadinessFlapping",
						Annotations: map[string]string{
							"description": "The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping",
							"summary":     "Node readiness status is flapping.",
						},
						Expr:   `sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (cluster, node) > 2`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeletPlegDurationHigh",
						Annotations: map[string]string{
							"description": "The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletplegdurationhigh",
							"summary":     "Kubelet Pod Lifecycle Event Generator is taking too long to relist.",
						},
						Expr:   `node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10`,
						For:    "5m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeletPodStartUpLatencyHigh",
						Annotations: map[string]string{
							"description": "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh",
							"summary":     "Kubelet Pod startup latency is too high.",
						},
						Expr:   `histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeletClientCertificateExpiration",
						Annotations: map[string]string{
							"description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration",
							"summary":     "Kubelet client certificate is about to expire.",
						},
						Expr:   "kubelet_certificate_manager_client_ttl_seconds < 604800",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeletClientCertificateExpiration",
						Annotations: map[string]string{
							"description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration",
							"summary":     "Kubelet client certificate is about to expire.",
						},
						Expr:   "kubelet_certificate_manager_client_ttl_seconds < 86400",
						Labels: map[string]string{"severity": "critical"},
					}, {
						Alert: "KubeletServerCertificateExpiration",
						Annotations: map[string]string{
							"description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration",
							"summary":     "Kubelet server certificate is about to expire.",
						},
						Expr:   "kubelet_certificate_manager_server_ttl_seconds < 604800",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeletServerCertificateExpiration",
						Annotations: map[string]string{
							"description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration",
							"summary":     "Kubelet server certificate is about to expire.",
						},
						Expr:   "kubelet_certificate_manager_server_ttl_seconds < 86400",
						Labels: map[string]string{"severity": "critical"},
					}, {
						Alert: "KubeletClientCertificateRenewalErrors",
						Annotations: map[string]string{
							"description": "Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificaterenewalerrors",
							"summary":     "Kubelet has failed to renew its client certificate.",
						},
						Expr:   "increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0",
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeletServerCertificateRenewalErrors",
						Annotations: map[string]string{
							"description": "Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificaterenewalerrors",
							"summary":     "Kubelet has failed to renew its server certificate.",
						},
						Expr:   "increase(kubelet_server_expiration_renew_errors[5m]) > 0",
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeletDown",
						Annotations: map[string]string{
							"description": "Kubelet has disappeared from Prometheus target discovery.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletdown",
							"summary":     "Target disappeared from Prometheus target discovery.",
						},
						Expr:   `absent(up{job="kubelet", metrics_path="/metrics"} == 1)`,
						For:    "15m",
						Labels: map[string]string{"severity": "critical"},
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var KubernetesSystemAlertRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-kubernetes-system",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "kubernetes-system",
				Rules: []vmo.Rule{
					{
						Alert: "KubeVersionMismatch",
						Annotations: map[string]string{
							"description": "There are {{ $value }} different semantic versions of Kubernetes components running.",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch",
							"summary":     "Different semantic versions of Kubernetes components running.",
						},
						Expr:   `count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					}, {
						Alert: "KubeClientErrors",
						Annotations: map[string]string{
							"description": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'",
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors",
							"summary":     "Kubernetes API server client is experiencing errors.",
						},
						Expr: `
(sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (cluster, instance, job, namespace)
  /
sum(rate(rest_client_requests_total[5m])) by (cluster, instance, job, namespace))
> 0.01
`,
						For:    "15m",
						Labels: map[string]string{"severity": "warning"},
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var NodeNetworkAlertRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-node-network",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "node-network",
				Rules: []vmo.Rule{
					{
						Alert: "NodeNetworkInterfaceFlapping",
						Annotations: map[string]string{
							"description": `Network interface "{{ $labels.device }}" changing its up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}`,
							"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/general/nodenetworkinterfaceflapping",
							"summary":     "Network interface is often changing its status",
						},
						Expr:   `changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2`,
						For:    "2m",
						Labels: map[string]string{"severity": "warning"},
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}

var K8SNodeRules = &vmo.VMRule{
	ObjectMeta: metav1.ObjectMeta{
		Labels:    RulesLabels,
		Name:      "victoria-metrics-node.rules",
		Namespace: Single.Namespace,
	},
	Spec: vmo.VMRuleSpec{
		Groups: []vmo.RuleGroup{
			{
				Name: "node.rules",
				Rules: []vmo.Rule{
					{
						Expr: `
topk by(cluster, namespace, pod) (1,
  max by (cluster, node, namespace, pod) (
    label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)")
))
`,
						Record: "node_namespace_pod:kube_pod_info:",
					}, {
						Expr: `
count by (cluster, node) (
  node_cpu_seconds_total{mode="idle",job="node-exporter"}
  * on (namespace, pod) group_left(node)
  topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:)
)
`,
						Record: "node:node_num_cpu:sum",
					}, {
						Expr: `
sum(
  node_memory_MemAvailable_bytes{job="node-exporter"} or
  (
    node_memory_Buffers_bytes{job="node-exporter"} +
    node_memory_Cached_bytes{job="node-exporter"} +
    node_memory_MemFree_bytes{job="node-exporter"} +
    node_memory_Slab_bytes{job="node-exporter"}
  )
) by (cluster)
`,
						Record: ":node_memory_MemAvailable_bytes:sum",
					}, {
						Expr: `
avg by (cluster, node) (
  sum without (mode) (
    rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m])
  )
)
`,
						Record: "node:node_cpu_utilization:ratio_rate5m",
					}, {
						Expr: `
avg by (cluster) (
  node:node_cpu_utilization:ratio_rate5m
)
`,
						Record: "cluster:node_cpu:ratio_rate5m",
					},
				},
			},
		},
	},
	TypeMeta: metav1.TypeMeta{
		APIVersion: "operator.victoriametrics.com/vmo",
		Kind:       "VMRule",
	},
}
