diff --git a/examples/max-pods-per-node.yml b/examples/max-pods-per-node.yml new file mode 100644 index 0000000000..5890950f3e --- /dev/null +++ b/examples/max-pods-per-node.yml @@ -0,0 +1,22 @@ +apiVersion: "descheduler/v1alpha2" +kind: "DeschedulerPolicy" +profiles: + - name: ProfileName + pluginConfig: + - name: "MaxPodsPerNode" + args: + maxPods: 30 + namespaces: + include: + - "default" + - name: "DefaultEvictor" + args: + evictSystemCriticalPods: false + evictLocalStoragePods: true + plugins: + deschedule: + enabled: + - "MaxPodsPerNode" + filter: + enabled: + - "DefaultEvictor" diff --git a/pkg/descheduler/setupplugins.go b/pkg/descheduler/setupplugins.go index eff3d049ae..67086235d9 100644 --- a/pkg/descheduler/setupplugins.go +++ b/pkg/descheduler/setupplugins.go @@ -19,6 +19,7 @@ package descheduler import ( "sigs.k8s.io/descheduler/pkg/framework/pluginregistry" "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" + "sigs.k8s.io/descheduler/pkg/framework/plugins/maxpodspernode" "sigs.k8s.io/descheduler/pkg/framework/plugins/nodeutilization" "sigs.k8s.io/descheduler/pkg/framework/plugins/podlifetime" "sigs.k8s.io/descheduler/pkg/framework/plugins/removeduplicates" @@ -47,4 +48,5 @@ func RegisterDefaultPlugins(registry pluginregistry.Registry) { pluginregistry.Register(removepodsviolatingnodeaffinity.PluginName, removepodsviolatingnodeaffinity.New, &removepodsviolatingnodeaffinity.RemovePodsViolatingNodeAffinity{}, &removepodsviolatingnodeaffinity.RemovePodsViolatingNodeAffinityArgs{}, removepodsviolatingnodeaffinity.ValidateRemovePodsViolatingNodeAffinityArgs, removepodsviolatingnodeaffinity.SetDefaults_RemovePodsViolatingNodeAffinityArgs, registry) pluginregistry.Register(removepodsviolatingnodetaints.PluginName, removepodsviolatingnodetaints.New, &removepodsviolatingnodetaints.RemovePodsViolatingNodeTaints{}, &removepodsviolatingnodetaints.RemovePodsViolatingNodeTaintsArgs{}, removepodsviolatingnodetaints.ValidateRemovePodsViolatingNodeTaintsArgs, removepodsviolatingnodetaints.SetDefaults_RemovePodsViolatingNodeTaintsArgs, registry) pluginregistry.Register(removepodsviolatingtopologyspreadconstraint.PluginName, removepodsviolatingtopologyspreadconstraint.New, &removepodsviolatingtopologyspreadconstraint.RemovePodsViolatingTopologySpreadConstraint{}, &removepodsviolatingtopologyspreadconstraint.RemovePodsViolatingTopologySpreadConstraintArgs{}, removepodsviolatingtopologyspreadconstraint.ValidateRemovePodsViolatingTopologySpreadConstraintArgs, removepodsviolatingtopologyspreadconstraint.SetDefaults_RemovePodsViolatingTopologySpreadConstraintArgs, registry) + pluginregistry.Register(maxpodspernode.PluginName, maxpodspernode.New, &maxpodspernode.MaxPodsPerNode{}, &maxpodspernode.MaxPodsPerNodeArgs{}, maxpodspernode.ValidateMaxPodsPerNodeArgs, maxpodspernode.SetDefaults_MaxPodsPerNodeArgs, registry) } diff --git a/pkg/framework/plugins/maxpodspernode/defaults.go b/pkg/framework/plugins/maxpodspernode/defaults.go new file mode 100644 index 0000000000..42ad047dee --- /dev/null +++ b/pkg/framework/plugins/maxpodspernode/defaults.go @@ -0,0 +1,36 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package maxpodspernode + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +func addDefaultingFuncs(scheme *runtime.Scheme) error { + return RegisterDefaults(scheme) +} + +// SetDefaults_MaxPodsPerNodeArgs sets default values for MaxPodsPerNodeArgs +func SetDefaults_MaxPodsPerNodeArgs(obj runtime.Object) { + args := obj.(*MaxPodsPerNodeArgs) + if args.Namespaces == nil { + args.Namespaces = nil + } + if args.LabelSelector == nil { + args.LabelSelector = nil + } + if args.MaxPods == 0 { + args.MaxPods = 110 // default Kubernetes max pods per node + } +} diff --git a/pkg/framework/plugins/maxpodspernode/doc.go b/pkg/framework/plugins/maxpodspernode/doc.go new file mode 100644 index 0000000000..e00f22bde0 --- /dev/null +++ b/pkg/framework/plugins/maxpodspernode/doc.go @@ -0,0 +1,16 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:defaulter-gen=TypeMeta + +package maxpodspernode diff --git a/pkg/framework/plugins/maxpodspernode/maxpodspernode.go b/pkg/framework/plugins/maxpodspernode/maxpodspernode.go new file mode 100644 index 0000000000..32c92b560e --- /dev/null +++ b/pkg/framework/plugins/maxpodspernode/maxpodspernode.go @@ -0,0 +1,129 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package maxpodspernode + +import ( + "context" + "fmt" + "sort" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + + "sigs.k8s.io/descheduler/pkg/descheduler/evictions" + podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" + frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" +) + +const PluginName = "MaxPodsPerNode" + +// MaxPodsPerNode evicts pods from nodes that exceed the configured maximum pod count +type MaxPodsPerNode struct { + logger klog.Logger + handle frameworktypes.Handle + args *MaxPodsPerNodeArgs + podFilter podutil.FilterFunc +} + +var _ frameworktypes.DeschedulePlugin = &MaxPodsPerNode{} + +// New builds plugin from its arguments while passing a handle +func New(ctx context.Context, args runtime.Object, handle frameworktypes.Handle) (frameworktypes.Plugin, error) { + maxPodsArgs, ok := args.(*MaxPodsPerNodeArgs) + if !ok { + return nil, fmt.Errorf("want args to be of type MaxPodsPerNodeArgs, got %T", args) + } + logger := klog.FromContext(ctx).WithValues("plugin", PluginName) + + var includedNamespaces, excludedNamespaces sets.Set[string] + if maxPodsArgs.Namespaces != nil { + includedNamespaces = sets.New(maxPodsArgs.Namespaces.Include...) + excludedNamespaces = sets.New(maxPodsArgs.Namespaces.Exclude...) + } + + // We can combine Filter and PreEvictionFilter since for this strategy it does not matter where we run PreEvictionFilter + podFilter, err := podutil.NewOptions(). + WithNamespaces(includedNamespaces). + WithoutNamespaces(excludedNamespaces). + WithLabelSelector(maxPodsArgs.LabelSelector). + BuildFilterFunc() + if err != nil { + return nil, fmt.Errorf("error initializing pod filter function: %v", err) + } + + return &MaxPodsPerNode{ + logger: logger, + handle: handle, + podFilter: podFilter, + args: maxPodsArgs, + }, nil +} + +// Name retrieves the plugin name +func (d *MaxPodsPerNode) Name() string { + return PluginName +} + +// Deschedule extension point implementation for the plugin +func (d *MaxPodsPerNode) Deschedule(ctx context.Context, nodes []*v1.Node) *frameworktypes.Status { + logger := klog.FromContext(klog.NewContext(ctx, d.logger)).WithValues("ExtensionPoint", frameworktypes.DescheduleExtensionPoint) + + for _, node := range nodes { + logger.V(2).Info("Processing node", "node", klog.KObj(node)) + + pods, err := podutil.ListAllPodsOnANode(node.Name, d.handle.GetPodsAssignedToNodeFunc(), d.podFilter) + if err != nil { + return &frameworktypes.Status{ + Err: fmt.Errorf("error listing pods on a node: %v", err), + } + } + + podCount := len(pods) + if podCount <= d.args.MaxPods { + logger.V(2).Info("Node pod count within limit", "node", klog.KObj(node), "podCount", podCount, "maxPods", d.args.MaxPods) + continue + } + + excessPods := podCount - d.args.MaxPods + logger.V(1).Info("Node exceeds max pods limit", "node", klog.KObj(node), "podCount", podCount, "maxPods", d.args.MaxPods, "excessPods", excessPods) + + // Sort pods by creation timestamp (newest first) so we evict the most recently created pods + sort.Slice(pods, func(i, j int) bool { + return pods[j].CreationTimestamp.Before(&pods[i].CreationTimestamp) + }) + + evicted := 0 + loop: + for i := 0; i < len(pods) && evicted < excessPods; i++ { + err := d.handle.Evictor().Evict(ctx, pods[i], evictions.EvictOptions{StrategyName: PluginName}) + if err == nil { + evicted++ + continue + } + switch err.(type) { + case *evictions.EvictionNodeLimitError: + break loop + case *evictions.EvictionTotalLimitError: + return nil + default: + logger.Error(err, "eviction failed", "pod", klog.KObj(pods[i])) + } + } + + logger.V(1).Info("Evicted pods from node", "node", klog.KObj(node), "evictedCount", evicted) + } + return nil +} diff --git a/pkg/framework/plugins/maxpodspernode/register.go b/pkg/framework/plugins/maxpodspernode/register.go new file mode 100644 index 0000000000..8bbf120632 --- /dev/null +++ b/pkg/framework/plugins/maxpodspernode/register.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package maxpodspernode + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +var ( + SchemeBuilder = runtime.NewSchemeBuilder() + localSchemeBuilder = &SchemeBuilder + AddToScheme = localSchemeBuilder.AddToScheme +) + +func init() { + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + localSchemeBuilder.Register(addDefaultingFuncs) +} diff --git a/pkg/framework/plugins/maxpodspernode/types.go b/pkg/framework/plugins/maxpodspernode/types.go new file mode 100644 index 0000000000..fc3c34cd8a --- /dev/null +++ b/pkg/framework/plugins/maxpodspernode/types.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package maxpodspernode + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/descheduler/pkg/api" +) + +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// MaxPodsPerNodeArgs holds arguments used to configure the MaxPodsPerNode plugin. +type MaxPodsPerNodeArgs struct { + metav1.TypeMeta `json:",inline"` + + Namespaces *api.Namespaces `json:"namespaces,omitempty"` + LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` + // MaxPods is the maximum number of pods allowed on a node. + // Pods will be evicted from nodes exceeding this count. + MaxPods int `json:"maxPods"` +} diff --git a/pkg/framework/plugins/maxpodspernode/validation.go b/pkg/framework/plugins/maxpodspernode/validation.go new file mode 100644 index 0000000000..f09d69ed74 --- /dev/null +++ b/pkg/framework/plugins/maxpodspernode/validation.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package maxpodspernode + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +// ValidateMaxPodsPerNodeArgs validates MaxPodsPerNode arguments +func ValidateMaxPodsPerNodeArgs(obj runtime.Object) error { + args := obj.(*MaxPodsPerNodeArgs) + var allErrs []error + + if args.MaxPods <= 0 { + allErrs = append(allErrs, fmt.Errorf("maxPods must be a positive integer")) + } + + // At most one of include/exclude can be set + if args.Namespaces != nil && len(args.Namespaces.Include) > 0 && len(args.Namespaces.Exclude) > 0 { + allErrs = append(allErrs, fmt.Errorf("only one of Include/Exclude namespaces can be set")) + } + + if args.LabelSelector != nil { + if _, err := metav1.LabelSelectorAsSelector(args.LabelSelector); err != nil { + allErrs = append(allErrs, fmt.Errorf("failed to get label selectors from strategy's params: %+v", err)) + } + } + + return utilerrors.NewAggregate(allErrs) +} diff --git a/pkg/framework/plugins/maxpodspernode/zz_generated.deepcopy.go b/pkg/framework/plugins/maxpodspernode/zz_generated.deepcopy.go new file mode 100644 index 0000000000..bb09a2a127 --- /dev/null +++ b/pkg/framework/plugins/maxpodspernode/zz_generated.deepcopy.go @@ -0,0 +1,63 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package maxpodspernode + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + api "sigs.k8s.io/descheduler/pkg/api" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MaxPodsPerNodeArgs) DeepCopyInto(out *MaxPodsPerNodeArgs) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = new(api.Namespaces) + (*in).DeepCopyInto(*out) + } + if in.LabelSelector != nil { + in, out := &in.LabelSelector, &out.LabelSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MaxPodsPerNodeArgs. +func (in *MaxPodsPerNodeArgs) DeepCopy() *MaxPodsPerNodeArgs { + if in == nil { + return nil + } + out := new(MaxPodsPerNodeArgs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MaxPodsPerNodeArgs) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/framework/plugins/maxpodspernode/zz_generated.defaults.go b/pkg/framework/plugins/maxpodspernode/zz_generated.defaults.go new file mode 100644 index 0000000000..3a5caa350e --- /dev/null +++ b/pkg/framework/plugins/maxpodspernode/zz_generated.defaults.go @@ -0,0 +1,33 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package maxpodspernode + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + return nil +} diff --git a/test/e2e/e2e_maxpodspernode_test.go b/test/e2e/e2e_maxpodspernode_test.go new file mode 100644 index 0000000000..af2bbf74ce --- /dev/null +++ b/test/e2e/e2e_maxpodspernode_test.go @@ -0,0 +1,252 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "os" + "strings" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + componentbaseconfig "k8s.io/component-base/config" + + "sigs.k8s.io/descheduler/pkg/api" + apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" + "sigs.k8s.io/descheduler/pkg/descheduler/client" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" + "sigs.k8s.io/descheduler/pkg/framework/plugins/maxpodspernode" +) + +func maxPodsPerNodePolicy(maxPodsArgs *maxpodspernode.MaxPodsPerNodeArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy { + return &apiv1alpha2.DeschedulerPolicy{ + Profiles: []apiv1alpha2.DeschedulerProfile{ + { + Name: maxpodspernode.PluginName + "Profile", + PluginConfigs: []apiv1alpha2.PluginConfig{ + { + Name: maxpodspernode.PluginName, + Args: runtime.RawExtension{ + Object: maxPodsArgs, + }, + }, + { + Name: defaultevictor.PluginName, + Args: runtime.RawExtension{ + Object: evictorArgs, + }, + }, + }, + Plugins: apiv1alpha2.Plugins{ + Filter: apiv1alpha2.PluginSet{ + Enabled: []string{ + defaultevictor.PluginName, + }, + }, + Deschedule: apiv1alpha2.PluginSet{ + Enabled: []string{ + maxpodspernode.PluginName, + }, + }, + }, + }, + }, + } +} + +func TestMaxPodsPerNode(t *testing.T) { + ctx := context.Background() + + clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") + if err != nil { + t.Errorf("Error during kubernetes client creation with %v", err) + } + + t.Log("Creating testing namespace") + testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}} + if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil { + t.Fatalf("Unable to create ns %v", testNamespace.Name) + } + defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) + + tests := []struct { + name string + replicaCount int32 + maxPods int + expectedEvictedPodCount int + }{ + { + name: "test-no-eviction-under-limit", + replicaCount: 2, + maxPods: 5, + expectedEvictedPodCount: 0, + }, + { + name: "test-eviction-over-limit", + replicaCount: 6, + maxPods: 3, + expectedEvictedPodCount: 3, + }, + { + name: "test-no-eviction-at-limit", + replicaCount: 4, + maxPods: 4, + expectedEvictedPodCount: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testLabel := map[string]string{"test": tc.name, "name": tc.name} + + deploymentObj := buildTestDeployment(tc.name, testNamespace.Name, tc.replicaCount, testLabel, nil) + + t.Logf("Creating deployment %v with %d replicas", deploymentObj.Name, tc.replicaCount) + _, err = clientSet.AppsV1().Deployments(deploymentObj.Namespace).Create(ctx, deploymentObj, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error creating deployment: %v", err) + } + defer func() { + clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{}) + waitForPodsToDisappear(ctx, t, clientSet, testLabel, testNamespace.Name) + }() + + t.Logf("Waiting for %d pods to be running", tc.replicaCount) + waitForPodsRunning(ctx, t, clientSet, testLabel, int(tc.replicaCount), testNamespace.Name) + + preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...) + + evictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + maxPodsArgs := &maxpodspernode.MaxPodsPerNodeArgs{ + MaxPods: tc.maxPods, + Namespaces: &api.Namespaces{ + Include: []string{testNamespace.Name}, + }, + } + + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(maxPodsPerNodePolicy(maxPodsArgs, evictorArgs)) + if err != nil { + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + + t.Logf("Creating %q policy CM with MaxPodsPerNode configured...", deschedulerPolicyConfigMapObj.Name) + _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + + defer func() { + t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) + err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + }() + + deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) + t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) + _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + + deschedulerPodName := "" + defer func() { + if deschedulerPodName != "" { + printPodLogs(ctx, t, clientSet, deschedulerPodName) + } + + t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) + err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + + waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) + }() + + t.Logf("Waiting for the descheduler pod running") + deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) + if len(deschedulerPods) != 0 { + deschedulerPodName = deschedulerPods[0].Name + } + + var meetsExpectations bool + var actualEvictedPodCount int + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + currentRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...) + actualEvictedPod := preRunNames.Difference(currentRunNames) + actualEvictedPodCount = actualEvictedPod.Len() + t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", + preRunNames.List(), currentRunNames.List(), actualEvictedPodCount) + if actualEvictedPodCount < tc.expectedEvictedPodCount { + t.Logf("Expecting at least %v pods evicted, got %v instead", tc.expectedEvictedPodCount, actualEvictedPodCount) + return false, nil + } + meetsExpectations = true + return true, nil + }); err != nil { + t.Errorf("Error waiting for fedscheduler running: %v", err) + } + + if !meetsExpectations { + t.Errorf("Unexpected number of pods have been evicted, got %v, expected at least %v", + actualEvictedPodCount, tc.expectedEvictedPodCount) + } else { + t.Logf("Total of %d pods were evicted for %s", actualEvictedPodCount, tc.name) + } + }) + } +} + +func waitForPodCount(ctx context.Context, t *testing.T, clientSet clientset.Interface, namespace string, labelMap map[string]string, desiredCount int) { + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labelMap).String(), + }) + if err != nil { + return false, err + } + + runningCount := 0 + for _, pod := range podList.Items { + if pod.Status.Phase == v1.PodRunning { + runningCount++ + } + } + + if runningCount != desiredCount { + t.Logf("Waiting for %d running pods, currently %d", desiredCount, runningCount) + return false, nil + } + return true, nil + }); err != nil { + t.Fatalf("Error waiting for pod count: %v", err) + } +}