/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpumanager

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/go-logr/logr"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	resourcehelper "k8s.io/component-helpers/resource"
	"k8s.io/klog/v2"
	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
	v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
	"k8s.io/kubernetes/pkg/features"
	"k8s.io/kubernetes/pkg/kubelet/cm/admission"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
	cmqos "k8s.io/kubernetes/pkg/kubelet/cm/qos"
	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
	"k8s.io/kubernetes/pkg/kubelet/managed"
	"k8s.io/kubernetes/pkg/kubelet/metrics"
	"k8s.io/utils/cpuset"
)

const (
	// PolicyStatic is the name of the static policy.
	// Should options be given, these will be ignored and backward (up to 1.21 included)
	// compatible behaviour will be enforced
	PolicyStatic policyName = "static"
	// ErrorSMTAlignment represents the type of an SMTAlignmentError
	ErrorSMTAlignment = "SMTAlignmentError"
)

// SMTAlignmentError represents an error due to SMT alignment
type SMTAlignmentError struct {
	RequestedCPUs         int
	CpusPerCore           int
	AvailablePhysicalCPUs int
	CausedByPhysicalCPUs  bool
}

func (e SMTAlignmentError) Error() string {
	if e.CausedByPhysicalCPUs {
		return fmt.Sprintf("SMT Alignment Error: not enough free physical CPUs: available physical CPUs = %d, requested CPUs = %d, CPUs per core = %d", e.AvailablePhysicalCPUs, e.RequestedCPUs, e.CpusPerCore)
	}
	return fmt.Sprintf("SMT Alignment Error: requested %d cpus not multiple cpus per core = %d", e.RequestedCPUs, e.CpusPerCore)
}

// Type returns human-readable type of this error. Used in the admission control to populate Admission Failure reason.
func (e SMTAlignmentError) Type() string {
	return ErrorSMTAlignment
}

// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
//
// This policy allocates CPUs exclusively for a container if all the following
// conditions are met:
//
// - The pod QoS class is Guaranteed.
// - The CPU request is a positive integer.
//
// The static policy maintains the following sets of logical CPUs:
//
//   - SHARED: Burstable, BestEffort, and non-integral Guaranteed containers
//     run here. Initially this contains all CPU IDs on the system. As
//     exclusive allocations are created and destroyed, this CPU set shrinks
//     and grows, accordingly. This is stored in the state as the default
//     CPU set.
//
//   - RESERVED: A subset of the shared pool which is not exclusively
//     allocatable. The membership of this pool is static for the lifetime of
//     the Kubelet. The size of the reserved pool is
//     ceil(systemreserved.cpu + kubereserved.cpu).
//     Reserved CPUs are taken topologically starting with lowest-indexed
//     physical core, as reported by cAdvisor.
//
//   - ASSIGNABLE: Equal to SHARED - RESERVED. Exclusive CPUs are allocated
//     from this pool.
//
//   - EXCLUSIVE ALLOCATIONS: CPU sets assigned exclusively to one container.
//     These are stored as explicit assignments in the state.
//
// When an exclusive allocation is made, the static policy also updates the
// default cpuset in the state abstraction. The CPU manager's periodic
// reconcile loop takes care of rewriting the cpuset in cgroupfs for any
// containers that may be running in the shared pool. For this reason,
// applications running within exclusively-allocated containers must tolerate
// potentially sharing their allocated CPUs for up to the CPU manager
// reconcile period.
type staticPolicy struct {
	// cpu socket topology
	topology *topology.CPUTopology
	// set of CPUs that is not available for exclusive assignment
	reservedCPUs cpuset.CPUSet
	// Superset of reservedCPUs. It includes not just the reservedCPUs themselves,
	// but also any siblings of those reservedCPUs on the same physical die.
	// NOTE: If the reserved set includes full physical CPUs from the beginning
	// (e.g. only reserved pairs of core siblings) this set is expected to be
	// identical to the reserved set.
	reservedPhysicalCPUs cpuset.CPUSet
	// topology manager reference to get container Topology affinity
	affinity topologymanager.Store
	// set of CPUs to reuse across allocations in a pod
	cpusToReuse map[string]cpuset.CPUSet
	// options allow to fine-tune the behaviour of the policy
	options StaticPolicyOptions
	// we compute this value multiple time, and it's not supposed to change
	// at runtime - the cpumanager can't deal with runtime topology changes anyway.
	cpuGroupSize int
}

// Ensure staticPolicy implements Policy interface
var _ Policy = &staticPolicy{}

// NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
func NewStaticPolicy(logger logr.Logger, topology *topology.CPUTopology, numReservedCPUs int, reservedCPUs cpuset.CPUSet, affinity topologymanager.Store, cpuPolicyOptions map[string]string) (Policy, error) {
	opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
	if err != nil {
		return nil, err
	}
	err = ValidateStaticPolicyOptions(opts, topology, affinity)
	if err != nil {
		return nil, err
	}

	cpuGroupSize := topology.CPUsPerCore()
	logger.Info("created with configuration", "options", opts, "cpuGroupSize", cpuGroupSize)

	policy := &staticPolicy{
		topology:     topology,
		affinity:     affinity,
		cpusToReuse:  make(map[string]cpuset.CPUSet),
		options:      opts,
		cpuGroupSize: cpuGroupSize,
	}

	allCPUs := topology.CPUDetails.CPUs()
	var reserved cpuset.CPUSet
	if reservedCPUs.Size() > 0 {
		reserved = reservedCPUs
	} else {
		// takeByTopology allocates CPUs associated with low-numbered cores from
		// allCPUs.
		//
		// For example: Given a system with 8 CPUs available and HT enabled,
		// if numReservedCPUs=2, then reserved={0,4}
		reserved, _ = policy.takeByTopology(logger, allCPUs, numReservedCPUs)
	}

	if reserved.Size() != numReservedCPUs {
		err := fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of %s did not equal %d)", reserved, numReservedCPUs)
		return nil, err
	}

	var reservedPhysicalCPUs cpuset.CPUSet
	for _, cpu := range reserved.UnsortedList() {
		core, err := topology.CPUCoreID(cpu)
		if err != nil {
			return nil, fmt.Errorf("[cpumanager] unable to build the reserved physical CPUs from the reserved set: %w", err)
		}
		reservedPhysicalCPUs = reservedPhysicalCPUs.Union(topology.CPUDetails.CPUsInCores(core))
	}

	logger.Info("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved, "reservedPhysicalCPUs", reservedPhysicalCPUs)
	policy.reservedCPUs = reserved
	policy.reservedPhysicalCPUs = reservedPhysicalCPUs

	return policy, nil
}

func (p *staticPolicy) Name() string {
	return string(PolicyStatic)
}

func (p *staticPolicy) Start(logger logr.Logger, s state.State) error {
	if err := p.validateState(logger, s); err != nil {
		logger.Error(err, "invalid state, please drain node and remove policy state file")
		return err
	}
	p.initializeMetrics(logger, s)
	return nil
}

func (p *staticPolicy) validateState(logger logr.Logger, s state.State) error {
	tmpAssignments := s.GetCPUAssignments()
	tmpDefaultCPUset := s.GetDefaultCPUSet()

	allCPUs := p.topology.CPUDetails.CPUs()
	if p.options.StrictCPUReservation {
		allCPUs = allCPUs.Difference(p.reservedCPUs)
	}

	// Default cpuset cannot be empty when assignments exist
	if tmpDefaultCPUset.IsEmpty() {
		if len(tmpAssignments) != 0 {
			return fmt.Errorf("default cpuset cannot be empty")
		}
		// state is empty initialize
		s.SetDefaultCPUSet(allCPUs)
		logger.Info("Static policy initialized", "defaultCPUSet", allCPUs)
		if managed.IsEnabled() {
			defaultCpus := s.GetDefaultCPUSet().Difference(p.reservedCPUs)
			s.SetDefaultCPUSet(defaultCpus)
		}
		return nil
	}

	// State has already been initialized from file (is not empty)
	// 1. Check if the reserved cpuset is not part of default cpuset because:
	// - kube/system reserved have changed (increased) - may lead to some containers not being able to start
	// - user tampered with file
	if p.options.StrictCPUReservation {
		if !p.reservedCPUs.Intersection(tmpDefaultCPUset).IsEmpty() {
			return fmt.Errorf("some of strictly reserved cpus: %q are present in defaultCpuSet: %q",
				p.reservedCPUs.Intersection(tmpDefaultCPUset).String(), tmpDefaultCPUset.String())
		}
	} else {
		// 2. This only applies when managed mode is disabled. Active workload partitioning feature
		//    removes the reserved cpus from the default cpu mask on purpose.
		if !managed.IsEnabled() && !p.reservedCPUs.Intersection(tmpDefaultCPUset).Equals(p.reservedCPUs) {
			return fmt.Errorf("not all reserved cpus: \"%s\" are present in defaultCpuSet: \"%s\"",
				p.reservedCPUs.String(), tmpDefaultCPUset.String())
		}
	}

	// 2. Check if state for static policy is consistent
	for pod := range tmpAssignments {
		for container, cset := range tmpAssignments[pod] {
			// None of the cpu in DEFAULT cset should be in s.assignments
			if !tmpDefaultCPUset.Intersection(cset).IsEmpty() {
				return fmt.Errorf("pod: %s, container: %s cpuset: %q overlaps with default cpuset %q",
					pod, container, cset.String(), tmpDefaultCPUset.String())
			}
		}
	}

	// 3. It's possible that the set of available CPUs has changed since
	// the state was written. This can be due to for example
	// offlining a CPU when kubelet is not running. If this happens,
	// CPU manager will run into trouble when later it tries to
	// assign non-existent CPUs to containers. Validate that the
	// topology that was received during CPU manager startup matches with
	// the set of CPUs stored in the state.
	totalKnownCPUs := tmpDefaultCPUset.Clone()
	tmpCPUSets := []cpuset.CPUSet{}
	for pod := range tmpAssignments {
		for _, cset := range tmpAssignments[pod] {
			tmpCPUSets = append(tmpCPUSets, cset)
		}
	}
	if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResourceManagers) {
		for _, podAssigments := range s.GetPodCPUAssignments() {
			tmpCPUSets = append(tmpCPUSets, podAssigments.CPUSet)
		}
	}
	totalKnownCPUs = totalKnownCPUs.Union(tmpCPUSets...)
	availableCPUs := p.topology.CPUDetails.CPUs()

	// CPU (workload) partitioning removes reserved cpus
	// from the default mask intentionally
	if managed.IsEnabled() {
		availableCPUs = availableCPUs.Difference(p.reservedCPUs)
	}

	if !totalKnownCPUs.Equals(availableCPUs) {
		return fmt.Errorf("current set of available CPUs \"%s\" doesn't match with CPUs in state \"%s\"",
			availableCPUs.String(), totalKnownCPUs.String())
	}

	return nil
}

// GetAllocatableCPUs returns the total set of CPUs available for allocation.
func (p *staticPolicy) GetAllocatableCPUs(s state.State) cpuset.CPUSet {
	return p.topology.CPUDetails.CPUs().Difference(p.reservedCPUs)
}

// GetAvailableCPUs returns the set of unassigned CPUs minus the reserved set.
func (p *staticPolicy) GetAvailableCPUs(s state.State) cpuset.CPUSet {
	return s.GetDefaultCPUSet().Difference(p.reservedCPUs)
}

func (p *staticPolicy) GetAvailablePhysicalCPUs(s state.State) cpuset.CPUSet {
	return s.GetDefaultCPUSet().Difference(p.reservedPhysicalCPUs)
}

func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, cset cpuset.CPUSet) {
	// If pod entries to m.cpusToReuse other than the current pod exist, delete them.
	for podUID := range p.cpusToReuse {
		if podUID != string(pod.UID) {
			delete(p.cpusToReuse, podUID)
		}
	}
	// If no cpuset exists for cpusToReuse by this pod yet, create one.
	if _, ok := p.cpusToReuse[string(pod.UID)]; !ok {
		p.cpusToReuse[string(pod.UID)] = cpuset.New()
	}
	// Check if the container is an init container.
	// If so, add its cpuset to the cpuset of reusable CPUs for any new allocations.
	for _, initContainer := range pod.Spec.InitContainers {
		if container.Name == initContainer.Name {
			if podutil.IsRestartableInitContainer(&initContainer) {
				// If the container is a restartable init container, we should not
				// reuse its cpuset, as a restartable init container can run with
				// regular containers.
				break
			}
			p.cpusToReuse[string(pod.UID)] = p.cpusToReuse[string(pod.UID)].Union(cset)
			return
		}
	}
	// Otherwise it is an app container.
	// Remove its cpuset from the cpuset of reusable CPUs for any new allocations.
	p.cpusToReuse[string(pod.UID)] = p.cpusToReuse[string(pod.UID)].Difference(cset)
}

// validatePodScopeResources checks for the "empty shared pool" scenario. This occurs
// when the sum of exclusive container CPU requests consumes the entire pod-level
// budget, leaving no CPUs for containers that require a shared pool. Such a
// configuration is invalid because it would lead to containers in the shared
// pool having an empty cpuset, causing them to run on the node's shared pool
// and breaking NUMA affinity.
func (p *staticPolicy) validatePodScopeResources(logger logr.Logger, pod *v1.Pod) error {
	podTotalCPUs := p.podGuaranteedCPUs(logger, pod)

	hasSharedLongRunningContainers := false
	// Sum CPU requests for all containers that run for the full pod lifetime
	// (main containers and restartable init containers) to determine the
	// total exclusive CPU usage that might impact the shared pool.
	sumOfLongRunningExclusiveCPUs := 0

	// Check for empty shared pool for standard init containers.
	for _, container := range pod.Spec.InitContainers {
		guaranteedCPUs := p.guaranteedCPUs(logger, pod, &container)

		if !podutil.IsRestartableInitContainer(&container) {
			if guaranteedCPUs == 0 && sumOfLongRunningExclusiveCPUs >= podTotalCPUs {
				// This check ensures that if there are any standard init containers that need a
				// shared pool, that pool is not empty. An empty pool would occur if the pod
				// budget is fully consumed by sidecars.
				return fmt.Errorf("pod rejected, pod has shared init containers but no cpus available for them")
			}
		} else {
			// If there are restartable init containers, keep track for the empty shared pool check.
			if guaranteedCPUs == 0 {
				hasSharedLongRunningContainers = true
			} else {
				sumOfLongRunningExclusiveCPUs += guaranteedCPUs
			}
		}
	}

	// Check for empty shared pool for concurrently running containers (app and restartable init containers).
	for _, container := range pod.Spec.Containers {
		guaranteedCPUs := p.guaranteedCPUs(logger, pod, &container)

		if guaranteedCPUs == 0 {
			hasSharedLongRunningContainers = true
		} else {
			sumOfLongRunningExclusiveCPUs += guaranteedCPUs
		}
	}

	// This check ensures that if there are any standard containers that need a shared
	// pool, that pool is not empty. An empty pool would occur if the pod budget is
	// fully consumed by sidecars and main containers with exclusive resources.
	if hasSharedLongRunningContainers && sumOfLongRunningExclusiveCPUs >= podTotalCPUs {
		return fmt.Errorf("pod rejected, sum of exclusive container cpu requests equals pod budget, leaving no cpus for shared containers")
	}

	return nil
}

// This function is the entry point for pod-level resource allocation.
// It's called once per pod by the Topology Manager's pod-scope admit handler.
// The logic here allocates a single "bubble" of CPUs for the entire pod
// and then partitions that bubble among the containers.
func (p *staticPolicy) AllocatePod(logger logr.Logger, s state.State, pod *v1.Pod) (rerr error) {
	podUID := string(pod.UID)
	logger = klog.LoggerWithValues(logger, "pod", klog.KObj(pod))
	logger.V(4).Info("AllocatePod called for pod-level managed pod")

	// 1. Calculate the total number of CPUs required for the pod, considering init container reuse.
	totalPodCPUs := p.podGuaranteedCPUs(logger, pod)
	if totalPodCPUs == 0 {
		// pod belongs in the shared pool (nothing to do; use default cpuset)
		return nil
	}
	logger.V(4).Info("Calculated total pod CPUs", "totalCPUs", totalPodCPUs)

	defer func() {
		if rerr != nil {
			metrics.CPUManagerPinningErrorsTotal.Inc()
			metrics.ResourceManagerAllocationErrorsTotal.WithLabelValues(metrics.ResourceManagerCPU, metrics.ResourceManagerPod).Inc()
		}
	}()

	// 2. Validate for the "empty shared pool" case.
	// Even though this is checked during hint generation, we must re-evaluate it here.
	// If the Topology Manager is running with the "best-effort" or "none" policies,
	// it will still admit the pod even if GetPodTopologyHints returns an empty hint list.
	// This ensures we explicitly reject the pod during allocation to prevent Kubelet
	// from attempting to create invalid cgroups (e.g., an empty cpuset.cpus) for shared containers.
	if err := p.validatePodScopeResources(logger, pod); err != nil {
		return admission.NewEmptyPodSharedPoolError(err)
	}

	// 3. Enforce SMT alignment policy if FullPhysicalCPUsOnly is enabled.
	if err := p.enforceSMTAlignment(s, totalPodCPUs); err != nil {
		return err
	}

	// 4. Allocate the entire CPU "bubble" for the pod using the hint from the Topology Manager.
	hint := p.affinity.GetAffinity(podUID, append(pod.Spec.InitContainers, pod.Spec.Containers...)[0].Name)
	podAllocation, err := p.allocateCPUs(logger, s, totalPodCPUs, hint.NUMANodeAffinity, cpuset.New())
	if err != nil {
		logger.Error(err, "Unable to allocate CPUs for pod", "totalPodCPUs", totalPodCPUs)
		return err
	}
	p.updateMetricsOnAllocate(logger, s, podAllocation)
	logger.V(4).Info("Allocated pod-level CPU bubble", "allocation", podAllocation.CPUs)

	// Store the pod-level allocation in the state.
	s.SetPodCPUSet(podUID, podAllocation.CPUs)

	// 5. Partition the pod's allocation, handling init container CPU reuse correctly.
	exclusiveCPUs := make(map[string]cpuset.CPUSet)
	sidecarCPUs := cpuset.New()

	// First, iterate through all init containers and allocate their CPUs from the initial pod bubble.
	for _, c := range pod.Spec.InitContainers {
		if numCPUs := p.guaranteedCPUs(logger, pod, &c); numCPUs > 0 {
			metrics.CPUManagerPinningRequestsTotal.Inc()
			// The pool available for this init container is the entire pod allocation
			// minus what's already taken by sidecars.
			runnablePool := podAllocation.CPUs.Difference(sidecarCPUs)
			cset, err := p.takeByTopology(logger, runnablePool, numCPUs)
			if err != nil {
				return err
			}
			metrics.ResourceManagerContainerAssignments.WithLabelValues(metrics.ResourceManagerCPU, metrics.ResourceManagerExclusivePod).Inc()
			exclusiveCPUs[c.Name] = cset

			// If it's a restartable sidecar, its CPUs are permanently consumed.
			if podutil.IsRestartableInitContainer(&c) {
				sidecarCPUs = sidecarCPUs.Union(cset)
			}
		} else {
			// Restartable init containers will access the main pod shared pool.
			if podutil.IsRestartableInitContainer(&c) {
				continue
			}

			// Non restartable init containers will have access to the whole pod pool
			// amount minus the sidecar exclusive CPUs up to that point.
			runnablePool := podAllocation.CPUs.Difference(sidecarCPUs)
			metrics.ResourceManagerContainerAssignments.WithLabelValues(metrics.ResourceManagerCPU, metrics.ResourceManagerSharedPod).Inc()
			exclusiveCPUs[c.Name] = runnablePool
		}
	}

	// Explicitly record that the full CPU bubble is available again, minus what sidecars are using.
	// This avoids carrying forward complex accounting from standard init containers.
	podSharedPool := podAllocation.CPUs.Difference(sidecarCPUs)

	// Second, iterate through regular containers, allocating from the remaining pool.
	for _, c := range pod.Spec.Containers {
		if numCPUs := p.guaranteedCPUs(logger, pod, &c); numCPUs > 0 {
			metrics.CPUManagerPinningRequestsTotal.Inc()
			cset, err := p.takeByTopology(logger, podSharedPool, numCPUs)
			if err != nil {
				return err
			}
			metrics.ResourceManagerContainerAssignments.WithLabelValues(metrics.ResourceManagerCPU, metrics.ResourceManagerExclusivePod).Inc()
			exclusiveCPUs[c.Name] = cset
			// Consume CPUs from the app container pool.
			podSharedPool = podSharedPool.Difference(cset)
		}
	}

	logger.V(4).Info("Partitioned pod-level CPU allocation", "exclusiveCPUs", podCPUAllocationToString(exclusiveCPUs), "podSharedPool", podSharedPool)

	// 6. Save all container assignments to the state.
	for _, c := range append(pod.Spec.InitContainers, pod.Spec.Containers...) {
		if cset, isExclusive := exclusiveCPUs[c.Name]; isExclusive {
			s.SetCPUSet(podUID, c.Name, cset)
		} else {
			s.SetCPUSet(podUID, c.Name, podSharedPool)
			metrics.ResourceManagerContainerAssignments.WithLabelValues(metrics.ResourceManagerCPU, metrics.ResourceManagerSharedPod).Inc()
		}

		metrics.ResourceManagerAllocationsTotal.WithLabelValues(metrics.ResourceManagerCPU, metrics.ResourceManagerPod).Inc()
	}

	return nil
}

func podCPUAllocationToString(alloc map[string]cpuset.CPUSet) string {
	if len(alloc) == 0 {
		return ""
	}
	var sb strings.Builder
	for name, cset := range alloc {
		fmt.Fprintf(&sb, ",%s={%s}", name, cset)
	}
	// if we gone so far, it means we have at least 1 alloc, so the cut is safe
	repr := sb.String()[1:]
	return "<" + repr + ">"
}

func (p *staticPolicy) enforceSMTAlignment(s state.State, numCPUs int) error {
	if !p.options.FullPhysicalCPUsOnly {
		return nil
	}

	if (numCPUs % p.cpuGroupSize) != 0 {
		// Since CPU Manager has been enabled requesting strict SMT alignment, it means a guaranteed pod can only be admitted
		// if the CPU requested is a multiple of the number of virtual cpus per physical cores.
		// In case CPU request is not a multiple of the number of virtual cpus per physical cores the Pod will be put
		// in Failed state, with SMTAlignmentError as reason. Since the allocation happens in terms of physical cores
		// and the scheduler is responsible for ensuring that the workload goes to a node that has enough CPUs,
		// the pod would be placed on a node where there are enough physical cores available to be allocated.
		// Just like the behaviour in case of static policy, takeByTopology will try to first allocate CPUs from the same socket
		// and only in case the request cannot be sattisfied on a single socket, CPU allocation is done for a workload to occupy all
		// CPUs on a physical core. Allocation of individual threads would never have to occur.
		return SMTAlignmentError{
			RequestedCPUs:        numCPUs,
			CpusPerCore:          p.cpuGroupSize,
			CausedByPhysicalCPUs: false,
		}
	}

	availablePhysicalCPUs := p.GetAvailablePhysicalCPUs(s).Size()

	// It's legal to reserve CPUs which are not core siblings. In this case the CPU allocator can descend to single cores
	// when picking CPUs. This will void the guarantee of FullPhysicalCPUsOnly. To prevent this, we need to additionally consider
	// all the core siblings of the reserved CPUs as unavailable when computing the free CPUs, before to start the actual allocation.
	// This way, by construction all possible CPUs allocation whose number is multiple of the SMT level are now correct again.
	if numCPUs > availablePhysicalCPUs {
		return SMTAlignmentError{
			RequestedCPUs:         numCPUs,
			CpusPerCore:           p.cpuGroupSize,
			AvailablePhysicalCPUs: availablePhysicalCPUs,
			CausedByPhysicalCPUs:  true,
		}
	}
	return nil
}

func (p *staticPolicy) Allocate(logger logr.Logger, s state.State, pod *v1.Pod, container *v1.Container) (rerr error) {
	logger = klog.LoggerWithValues(logger, "pod", klog.KObj(pod), "podUID", pod.UID, "containerName", container.Name)
	logger.Info("Allocate start") // V=0 for backward compatibility
	defer logger.V(2).Info("Allocate end")

	numCPUs := p.guaranteedCPUs(logger, pod, container)
	if numCPUs == 0 {
		// container belongs in the shared pool (nothing to do; use default cpuset)
		return nil
	}

	if (!utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResourceManagers) || !utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources)) && resourcehelper.IsPodLevelResourcesSet(pod) {
		logger.V(2).Info("CPU Manager allocation skipped, pod is using pod-level resources but the PodLevelResourceManagers feature gate is not enabled", "pod", klog.KObj(pod), "podUID", pod.UID)
		return nil
	}

	// container belongs in an exclusively allocated pool
	metrics.CPUManagerPinningRequestsTotal.Inc()
	defer func() {
		if rerr != nil {
			metrics.CPUManagerPinningErrorsTotal.Inc()
			metrics.ResourceManagerAllocationErrorsTotal.WithLabelValues(metrics.ResourceManagerCPU, metrics.ResourceManagerNode).Inc()
			if p.options.FullPhysicalCPUsOnly {
				metrics.ContainerAlignedComputeResourcesFailure.WithLabelValues(metrics.AlignScopeContainer, metrics.AlignedPhysicalCPU).Inc()
			}
			return
		}
		// TODO: move in updateMetricsOnAllocate
		if p.options.FullPhysicalCPUsOnly {
			// increment only if we know we allocate aligned resources
			metrics.ContainerAlignedComputeResources.WithLabelValues(metrics.AlignScopeContainer, metrics.AlignedPhysicalCPU).Inc()
		}
	}()

	if err := p.enforceSMTAlignment(s, numCPUs); err != nil {
		return err
	}

	if cset, ok := s.GetCPUSet(string(pod.UID), container.Name); ok {
		p.updateCPUsToReuse(pod, container, cset)
		logger.Info("Static policy: container already present in state, skipping")
		return nil
	}

	// Call Topology Manager to get the aligned socket affinity across all hint providers.
	hint := p.affinity.GetAffinity(string(pod.UID), container.Name)
	logger.Info("Topology Affinity", "affinity", hint)

	// Allocate CPUs according to the NUMA affinity contained in the hint.
	cpuAllocation, err := p.allocateCPUs(logger, s, numCPUs, hint.NUMANodeAffinity, p.cpusToReuse[string(pod.UID)])
	if err != nil {
		logger.Error(err, "Unable to allocate CPUs", "numCPUs", numCPUs)
		return err
	}

	s.SetCPUSet(string(pod.UID), container.Name, cpuAllocation.CPUs)
	p.updateCPUsToReuse(pod, container, cpuAllocation.CPUs)
	p.updateMetricsOnAllocate(logger, s, cpuAllocation)
	metrics.ResourceManagerAllocationsTotal.WithLabelValues(metrics.ResourceManagerCPU, metrics.ResourceManagerNode).Inc()
	metrics.ResourceManagerContainerAssignments.WithLabelValues(metrics.ResourceManagerCPU, metrics.ResourceManagerExclusiveNode).Inc()

	logger.V(4).Info("Allocated exclusive CPUs", "cpuset", cpuAllocation.CPUs.String())
	return nil
}

// getAssignedCPUsOfSiblings returns assigned cpus of given container's siblings(all containers other than the given container) in the given pod `podUID`.
func getAssignedCPUsOfSiblings(s state.State, podUID string, containerName string) cpuset.CPUSet {
	assignments := s.GetCPUAssignments()
	cset := cpuset.New()
	for name, cpus := range assignments[podUID] {
		if containerName == name {
			continue
		}
		cset = cset.Union(cpus)
	}
	return cset
}

func (p *staticPolicy) RemoveContainer(logger logr.Logger, s state.State, podUID string, containerName string) error {
	logger = klog.LoggerWithValues(logger, "podUID", podUID, "containerName", containerName)
	logger.Info("RemoveContainer start") // backward compatibility
	defer logger.V(4).Info("RemoveContainer start")
	toRelease, ok := s.GetCPUSet(podUID, containerName)
	if !ok {
		return nil
	}
	s.Delete(podUID, containerName)

	// Check if this pod is managed with pod-level CPUs.
	if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResourceManagers) {
		if podCPUSet, isPodLevel := s.GetPodCPUSet(podUID); isPodLevel {
			// If this was the last container for the pod, then release the entire pod-level CPU set.
			if len(s.GetCPUAssignments()[podUID]) == 0 {
				updatedCPUSets := s.GetDefaultCPUSet().Union(podCPUSet)
				s.SetDefaultCPUSet(updatedCPUSets)
				s.DeletePod(podUID) // Clean up all state for the pod.
				p.updateMetricsOnRelease(logger, s, podCPUSet)
				logger.Info("Released pod-level CPUs", "defaultCPUSet", updatedCPUSets)
			}
			// If other containers still exist, do not release any CPUs yet.
			// The pod-level CPUs will be released when the last container is removed.
			return nil
		}
	}

	cpusInUse := getAssignedCPUsOfSiblings(s, podUID, containerName)
	// Mutate the shared pool, adding released cpus.
	toRelease = toRelease.Difference(cpusInUse)
	updatedCPUs := s.GetDefaultCPUSet().Union(toRelease)
	s.SetDefaultCPUSet(updatedCPUs)
	p.updateMetricsOnRelease(logger, s, toRelease)
	logger.Info(" RemoveContainer end", "defaultCPUSet", updatedCPUs)
	return nil
}

func (p *staticPolicy) allocateCPUs(logger logr.Logger, s state.State, numCPUs int, numaAffinity bitmask.BitMask, reusableCPUs cpuset.CPUSet) (topology.Allocation, error) {
	logger.Info("AllocateCPUs", "numCPUs", numCPUs, "socket", numaAffinity)

	allocatableCPUs := p.GetAvailableCPUs(s).Union(reusableCPUs)

	// If there are aligned CPUs in numaAffinity, attempt to take those first.
	result := topology.EmptyAllocation()
	if numaAffinity != nil {
		alignedCPUs := p.getAlignedCPUs(numaAffinity, allocatableCPUs)

		numAlignedToAlloc := alignedCPUs.Size()
		if numCPUs < numAlignedToAlloc {
			numAlignedToAlloc = numCPUs
		}

		allocatedCPUs, err := p.takeByTopology(logger, alignedCPUs, numAlignedToAlloc)
		if err != nil {
			return topology.EmptyAllocation(), err
		}

		result.CPUs = result.CPUs.Union(allocatedCPUs)
	}

	// Get any remaining CPUs from what's leftover after attempting to grab aligned ones.
	remainingCPUs, err := p.takeByTopology(logger, allocatableCPUs.Difference(result.CPUs), numCPUs-result.CPUs.Size())
	if err != nil {
		return topology.EmptyAllocation(), err
	}
	result.CPUs = result.CPUs.Union(remainingCPUs)
	result.Aligned = p.topology.CheckAlignment(result.CPUs)

	// Remove allocated CPUs from the shared CPUSet.
	s.SetDefaultCPUSet(s.GetDefaultCPUSet().Difference(result.CPUs))

	logger.Info("AllocateCPUs", "result", result.String())
	return result, nil
}

func isIntegralCPUAmount(cpuQuantity resource.Quantity) bool {
	return cpuQuantity.Value()*1000 == cpuQuantity.MilliValue()
}

func (p *staticPolicy) guaranteedCPUs(logger logr.Logger, pod *v1.Pod, container *v1.Container) int {
	qos := v1qos.GetPodQOS(pod)
	if qos != v1.PodQOSGuaranteed {
		logger.V(5).Info("Exclusive CPU allocation skipped, pod QoS is not guaranteed", "qos", qos)
		return 0
	}

	cpuQuantity := container.Resources.Requests[v1.ResourceCPU]

	// For pod-level resource management, a container is only considered for exclusive
	// CPUs if its request equals its limit for both the CPU and Memory. This
	// aligns with the Guaranteed QoS requirement for container-level resources.
	if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResourceManagers) && resourcehelper.IsPodLevelResourcesSet(pod) && !cmqos.IsContainerEquivalentQOSGuaranteed(container) {
		logger.V(5).Info("Exclusive CPU allocation skipped, container is not eligible, request and limit are not equal", "pod", klog.KObj(pod), "containerName", container.Name)
		return 0
	}

	cpuValue := cpuQuantity.Value()
	if !isIntegralCPUAmount(cpuQuantity) {
		logger.V(5).Info("Exclusive CPU allocation skipped, container requested non-integral CPUs", "pod", klog.KObj(pod), "containerName", container.Name, "cpu", cpuValue)
		return 0
	}
	// Safe downcast to do for all systems with < 2.1 billion CPUs.
	// Per the language spec, `int` is guaranteed to be at least 32 bits wide.
	// https://golang.org/ref/spec#Numeric_types
	return int(cpuValue)
}

func (p *staticPolicy) podGuaranteedCPUs(logger logr.Logger, pod *v1.Pod) int {
	// If pod-level resources are set, use them directly.
	// This check is important because this function is called from GetPodTopologyHints,
	// which runs before the main feature gate check in AllocatePod.
	if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResourceManagers) && resourcehelper.IsPodLevelResourcesSet(pod) {
		qos := v1qos.GetPodQOS(pod)
		if qos != v1.PodQOSGuaranteed {
			return 0
		}

		if cpuQuantity, ok := pod.Spec.Resources.Requests[v1.ResourceCPU]; ok {
			// We only consider integer CPU requests for guaranteed CPUs at the pod level.
			if isIntegralCPUAmount(cpuQuantity) {
				return int(cpuQuantity.Value())
			}
		}
		return 0
	}

	// The maximum of requested CPUs by init containers.
	requestedByInitContainers := 0
	requestedByRestartableInitContainers := 0
	for _, container := range pod.Spec.InitContainers {
		if _, ok := container.Resources.Requests[v1.ResourceCPU]; !ok {
			continue
		}
		requestedCPU := p.guaranteedCPUs(logger, pod, &container)
		// See https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/753-sidecar-containers#resources-calculation-for-scheduling-and-pod-admission
		// for the detail.
		if podutil.IsRestartableInitContainer(&container) {
			requestedByRestartableInitContainers += requestedCPU
		} else if requestedByRestartableInitContainers+requestedCPU > requestedByInitContainers {
			requestedByInitContainers = requestedByRestartableInitContainers + requestedCPU
		}
	}

	// The sum of requested CPUs by app containers.
	requestedByAppContainers := 0
	for _, container := range pod.Spec.Containers {
		if _, ok := container.Resources.Requests[v1.ResourceCPU]; !ok {
			continue
		}
		requestedByAppContainers += p.guaranteedCPUs(logger, pod, &container)
	}

	requestedByLongRunningContainers := requestedByAppContainers + requestedByRestartableInitContainers
	if requestedByInitContainers > requestedByLongRunningContainers {
		return requestedByInitContainers
	}
	return requestedByLongRunningContainers
}

func (p *staticPolicy) takeByTopology(logger logr.Logger, availableCPUs cpuset.CPUSet, numCPUs int) (cpuset.CPUSet, error) {
	cpuSortingStrategy := CPUSortingStrategyPacked
	if p.options.DistributeCPUsAcrossCores {
		cpuSortingStrategy = CPUSortingStrategySpread
	}

	if p.options.DistributeCPUsAcrossNUMA {
		cpuGroupSize := 1
		if p.options.FullPhysicalCPUsOnly {
			cpuGroupSize = p.cpuGroupSize
		}
		return takeByTopologyNUMADistributed(logger, p.topology, availableCPUs, numCPUs, cpuGroupSize, cpuSortingStrategy)
	}

	return takeByTopologyNUMAPacked(logger, p.topology, availableCPUs, numCPUs, cpuSortingStrategy, p.options.PreferAlignByUncoreCacheOption)
}

func (p *staticPolicy) GetTopologyHints(logger logr.Logger, s state.State, pod *v1.Pod, container *v1.Container) map[string][]topologymanager.TopologyHint {
	logger = klog.LoggerWithValues(logger, "pod", klog.KObj(pod), "podUID", pod.UID, "containerName", container.Name)

	// Get a count of how many guaranteed CPUs have been requested.
	requested := p.guaranteedCPUs(logger, pod, container)

	// Number of required CPUs is not an integer or a container is not part of the Guaranteed QoS class.
	// It will be treated by the TopologyManager as having no preference and cause it to ignore this
	// resource when considering pod alignment.
	// In terms of hints, this is equal to: TopologyHints[NUMANodeAffinity: nil, Preferred: true].
	if requested == 0 {
		return nil
	}

	// If the pod has pod-level resources but the feature gate is disabled,
	// log it and return nil hints to admit the pod without alignment.
	if (!utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResourceManagers) || !utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources)) && resourcehelper.IsPodLevelResourcesSet(pod) {
		logger.V(3).Info("CPU Manager hint generation skipped, pod is using pod-level resources but the PodLevelResourceManagers feature gate is not enabled")
		return nil
	}

	// Short circuit to regenerate the same hints if there are already
	// guaranteed CPUs allocated to the Container. This might happen after a
	// kubelet restart, for example.
	if allocated, exists := s.GetCPUSet(string(pod.UID), container.Name); exists {
		if allocated.Size() != requested {
			logger.Info("CPUs already allocated to container with different number than request", "requestedSize", requested, "allocatedSize", allocated.Size())
			// An empty list of hints will be treated as a preference that cannot be satisfied.
			// In definition of hints this is equal to: TopologyHint[NUMANodeAffinity: nil, Preferred: false].
			// For all but the best-effort policy, the Topology Manager will throw a pod-admission error.
			return map[string][]topologymanager.TopologyHint{
				string(v1.ResourceCPU): {},
			}
		}
		logger.Info("Regenerating TopologyHints for CPUs already allocated")
		return map[string][]topologymanager.TopologyHint{
			string(v1.ResourceCPU): p.generateCPUTopologyHints(allocated, cpuset.New(), requested),
		}
	}

	// Get a list of available CPUs.
	available := p.GetAvailableCPUs(s)

	// Get a list of reusable CPUs (e.g. CPUs reused from initContainers).
	// It should be an empty CPUSet for a newly created pod.
	reusable := p.cpusToReuse[string(pod.UID)]

	// Generate hints.
	cpuHints := p.generateCPUTopologyHints(available, reusable, requested)
	logger.Info("TopologyHints generated", "cpuHints", cpuHints)

	return map[string][]topologymanager.TopologyHint{
		string(v1.ResourceCPU): cpuHints,
	}
}

func (p *staticPolicy) GetPodTopologyHints(logger logr.Logger, s state.State, pod *v1.Pod) map[string][]topologymanager.TopologyHint {
	logger = klog.LoggerWithValues(logger, "pod", klog.KObj(pod), "podUID", pod.UID)

	// Get a count of how many guaranteed CPUs have been requested by Pod.
	requested := p.podGuaranteedCPUs(logger, pod)

	// Number of required CPUs is not an integer or a pod is not part of the Guaranteed QoS class.
	// It will be treated by the TopologyManager as having no preference and cause it to ignore this
	// resource when considering pod alignment.
	// In terms of hints, this is equal to: TopologyHints[NUMANodeAffinity: nil, Preferred: true].
	if requested == 0 {
		return nil
	}

	// If the pod has pod-level resources but the feature gate is disabled,
	// log it and return nil hints to admit the pod without alignment.
	if (!utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResourceManagers) || !utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources)) && resourcehelper.IsPodLevelResourcesSet(pod) {
		logger.V(3).Info("CPU Manager pod hint generation skipped, pod is using pod-level resources but the PodLevelResourceManagers feature gate is not enabled", "podUID", pod.UID)
		return nil
	}

	// Validate that if a pod has containers that will be placed in a shared pool,
	// there are actually CPUs left over for that pool after accounting for all
	// exclusive allocations. If the sum of exclusive CPU requests consumes the
	// entire pod-level CPU budget, no hints will be generated, causing the pod
	// to be rejected by the Topology Manager.
	if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResourceManagers) && resourcehelper.IsPodLevelResourcesSet(pod) {
		if err := p.validatePodScopeResources(logger, pod); err != nil {
			logger.V(2).Info("Invalid pod spec. Sum of exclusive container requests equals pod budget, leaving no CPUs for shared containers")
			return map[string][]topologymanager.TopologyHint{
				string(v1.ResourceCPU): {},
			}
		}
	}

	assignedCPUs := cpuset.New()
	for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) {
		logger_ := klog.LoggerWithValues(logger, "containerName", container.Name)

		requestedByContainer := p.guaranteedCPUs(logger, pod, &container)
		// Short circuit to regenerate the same hints if there are already
		// guaranteed CPUs allocated to the Container. This might happen after a
		// kubelet restart, for example.
		if allocated, exists := s.GetCPUSet(string(pod.UID), container.Name); exists {
			if allocated.Size() != requestedByContainer {
				logger_.Info("CPUs already allocated to container with different number than request", "allocatedSize", requested, "requestedByContainer", requestedByContainer, "allocatedSize", allocated.Size())
				// An empty list of hints will be treated as a preference that cannot be satisfied.
				// In definition of hints this is equal to: TopologyHint[NUMANodeAffinity: nil, Preferred: false].
				// For all but the best-effort policy, the Topology Manager will throw a pod-admission error.
				return map[string][]topologymanager.TopologyHint{
					string(v1.ResourceCPU): {},
				}
			}
			// A set of CPUs already assigned to containers in this pod
			assignedCPUs = assignedCPUs.Union(allocated)
		}
	}
	if assignedCPUs.Size() == requested {
		logger.Info("Regenerating TopologyHints for CPUs already allocated")
		return map[string][]topologymanager.TopologyHint{
			string(v1.ResourceCPU): p.generateCPUTopologyHints(assignedCPUs, cpuset.New(), requested),
		}
	}

	// Get a list of available CPUs.
	available := p.GetAvailableCPUs(s)

	// Get a list of reusable CPUs (e.g. CPUs reused from initContainers).
	// It should be an empty CPUSet for a newly created pod.
	reusable := p.cpusToReuse[string(pod.UID)]

	// Ensure any CPUs already assigned to containers in this pod are included as part of the hint generation.
	reusable = reusable.Union(assignedCPUs)

	// Generate hints.
	cpuHints := p.generateCPUTopologyHints(available, reusable, requested)
	logger.Info("TopologyHints generated", "cpuHints", cpuHints)

	return map[string][]topologymanager.TopologyHint{
		string(v1.ResourceCPU): cpuHints,
	}
}

// generateCPUTopologyHints generates a set of TopologyHints given the set of
// available CPUs and the number of CPUs being requested.
//
// It follows the convention of marking all hints that have the same number of
// bits set as the narrowest matching NUMANodeAffinity with 'Preferred: true', and
// marking all others with 'Preferred: false'.
func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reusableCPUs cpuset.CPUSet, request int) []topologymanager.TopologyHint {
	// Initialize minAffinitySize to include all NUMA Nodes.
	minAffinitySize := p.topology.CPUDetails.NUMANodes().Size()

	// Iterate through all combinations of numa nodes bitmask and build hints from them.
	hints := []topologymanager.TopologyHint{}
	bitmask.IterateBitMasks(p.topology.CPUDetails.NUMANodes().List(), func(mask bitmask.BitMask) {
		// First, update minAffinitySize for the current request size.
		cpusInMask := p.topology.CPUDetails.CPUsInNUMANodes(mask.GetBits()...).Size()
		if cpusInMask >= request && mask.Count() < minAffinitySize {
			minAffinitySize = mask.Count()
		}

		// Then check to see if we have enough CPUs available on the current
		// numa node bitmask to satisfy the CPU request.
		numMatching := 0
		for _, c := range reusableCPUs.List() {
			// Disregard this mask if its NUMANode isn't part of it.
			if !mask.IsSet(p.topology.CPUDetails[c].NUMANodeID) {
				return
			}
			numMatching++
		}

		// Finally, check to see if enough available CPUs remain on the current
		// NUMA node combination to satisfy the CPU request.
		for _, c := range availableCPUs.List() {
			if mask.IsSet(p.topology.CPUDetails[c].NUMANodeID) {
				numMatching++
			}
		}

		// If they don't, then move onto the next combination.
		if numMatching < request {
			return
		}

		// Otherwise, create a new hint from the numa node bitmask and add it to the
		// list of hints.  We set all hint preferences to 'false' on the first
		// pass through.
		hints = append(hints, topologymanager.TopologyHint{
			NUMANodeAffinity: mask,
			Preferred:        false,
		})
	})

	// Loop back through all hints and update the 'Preferred' field based on
	// counting the number of bits sets in the affinity mask and comparing it
	// to the minAffinitySize. Only those with an equal number of bits set (and
	// with a minimal set of numa nodes) will be considered preferred.
	for i := range hints {
		if p.options.AlignBySocket && p.isHintSocketAligned(hints[i], minAffinitySize) {
			hints[i].Preferred = true
			continue
		}
		if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
			hints[i].Preferred = true
		}
	}

	return hints
}

// isHintSocketAligned function return true if numa nodes in hint are socket aligned.
func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool {
	numaNodesBitMask := hint.NUMANodeAffinity.GetBits()
	numaNodesPerSocket := p.topology.NumNUMANodes / p.topology.NumSockets
	if numaNodesPerSocket == 0 {
		return false
	}
	// minSockets refers to minimum number of socket required to satify allocation.
	// A hint is considered socket aligned if sockets across which numa nodes span is equal to minSockets
	minSockets := (minAffinitySize + numaNodesPerSocket - 1) / numaNodesPerSocket
	return p.topology.CPUDetails.SocketsInNUMANodes(numaNodesBitMask...).Size() == minSockets
}

// getAlignedCPUs return set of aligned CPUs based on numa affinity mask and configured policy options.
func (p *staticPolicy) getAlignedCPUs(numaAffinity bitmask.BitMask, allocatableCPUs cpuset.CPUSet) cpuset.CPUSet {
	alignedCPUs := cpuset.New()
	numaBits := numaAffinity.GetBits()

	// If align-by-socket policy option is enabled, NUMA based hint is expanded to
	// socket aligned hint. It will ensure that first socket aligned available CPUs are
	// allocated before we try to find CPUs across socket to satisfy allocation request.
	if p.options.AlignBySocket {
		socketBits := p.topology.CPUDetails.SocketsInNUMANodes(numaBits...).UnsortedList()
		for _, socketID := range socketBits {
			alignedCPUs = alignedCPUs.Union(allocatableCPUs.Intersection(p.topology.CPUDetails.CPUsInSockets(socketID)))
		}
		return alignedCPUs
	}

	for _, numaNodeID := range numaBits {
		alignedCPUs = alignedCPUs.Union(allocatableCPUs.Intersection(p.topology.CPUDetails.CPUsInNUMANodes(numaNodeID)))
	}

	return alignedCPUs
}

func (p *staticPolicy) initializeMetrics(logger logr.Logger, s state.State) {
	metrics.CPUManagerSharedPoolSizeMilliCores.Set(float64(p.GetAvailableCPUs(s).Size() * 1000))
	metrics.ContainerAlignedComputeResourcesFailure.WithLabelValues(metrics.AlignScopeContainer, metrics.AlignedPhysicalCPU).Add(0) // ensure the value exists
	metrics.ContainerAlignedComputeResources.WithLabelValues(metrics.AlignScopeContainer, metrics.AlignedPhysicalCPU).Add(0)        // ensure the value exists
	metrics.ContainerAlignedComputeResources.WithLabelValues(metrics.AlignScopeContainer, metrics.AlignedUncoreCache).Add(0)        // ensure the value exists
	totalAssignedCPUs := getTotalAssignedExclusiveCPUs(s)
	metrics.CPUManagerExclusiveCPUsAllocationCount.Set(float64(totalAssignedCPUs.Size()))
	updateAllocationPerNUMAMetric(logger, p.topology, totalAssignedCPUs)
}

func (p *staticPolicy) updateMetricsOnAllocate(logger logr.Logger, s state.State, cpuAlloc topology.Allocation) {
	ncpus := cpuAlloc.CPUs.Size()
	metrics.CPUManagerExclusiveCPUsAllocationCount.Add(float64(ncpus))
	metrics.CPUManagerSharedPoolSizeMilliCores.Add(float64(-ncpus * 1000))
	if cpuAlloc.Aligned.UncoreCache {
		metrics.ContainerAlignedComputeResources.WithLabelValues(metrics.AlignScopeContainer, metrics.AlignedUncoreCache).Inc()
	}
	totalAssignedCPUs := getTotalAssignedExclusiveCPUs(s)
	updateAllocationPerNUMAMetric(logger, p.topology, totalAssignedCPUs)
}

func (p *staticPolicy) updateMetricsOnRelease(logger logr.Logger, s state.State, cset cpuset.CPUSet) {
	ncpus := cset.Size()
	metrics.CPUManagerExclusiveCPUsAllocationCount.Add(float64(-ncpus))
	metrics.CPUManagerSharedPoolSizeMilliCores.Add(float64(ncpus * 1000))
	totalAssignedCPUs := getTotalAssignedExclusiveCPUs(s)
	updateAllocationPerNUMAMetric(logger, p.topology, totalAssignedCPUs.Difference(cset))
}

func getTotalAssignedExclusiveCPUs(s state.State) cpuset.CPUSet {
	totalAssignedCPUs := cpuset.New()
	for _, assignment := range s.GetCPUAssignments() {
		for _, cset := range assignment {
			totalAssignedCPUs = totalAssignedCPUs.Union(cset)
		}
	}
	return totalAssignedCPUs
}

func updateAllocationPerNUMAMetric(logger logr.Logger, topo *topology.CPUTopology, allocatedCPUs cpuset.CPUSet) {
	numaCount := make(map[int]int)

	// Count CPUs allocated per NUMA node
	for _, cpuID := range allocatedCPUs.UnsortedList() {
		numaNode, err := topo.CPUNUMANodeID(cpuID)
		if err != nil {
			//NOTE: We are logging the error but it is highly unlikely to happen as the CPUset
			//      is already computed, evaluated and there is no room for user tampering.
			logger.Error(err, "Unable to determine NUMA node", "cpuID", cpuID)
		}
		numaCount[numaNode]++
	}

	// Update metric
	for numaNode, count := range numaCount {
		metrics.CPUManagerAllocationPerNUMA.WithLabelValues(strconv.Itoa(numaNode)).Set(float64(count))
	}
}
