diff --git a/cmd/experimental/kueue-populator/README.md b/cmd/experimental/kueue-populator/README.md index 2459255b5e8..3c5fed9c761 100644 --- a/cmd/experimental/kueue-populator/README.md +++ b/cmd/experimental/kueue-populator/README.md @@ -68,18 +68,23 @@ To deploy the `kueue-populator` using a locally built image into a Kind cluster: ``` ## Configuration -The `kueue-populator` supports the following command-line flags: +The `kueue-populator` reads its configuration from the file passed to `--config`. +If `--config` is omitted, default values are used. +Logging is configured with standard Zap flags, such as `--zap-log-level`. -* `--local-queue-name`: The name of the `LocalQueue` to create by default in selected namespaces. Defaults to `"default"`. -* `--zap-log-level`: Sets the logging verbosity level for the Zap logger (e.g., `info`, `debug`, `error`). +The configuration supports the following fields: -To set these flags, modify the `args` list in the `Deployment` manifest (`config/manager/manager.yaml`) or use a Kustomize patch. For example: +* `localQueueName`: The name of the `LocalQueue` to create by default in selected namespaces. Defaults to `"default"`. +* `localQueueNameMode`: How to derive LocalQueue names. Use `Static` for `localQueueName` or `AsClusterQueue` to use each `ClusterQueue` name. +* `managedJobsNamespaceSelector`: Namespace selector used by the populator. The Helm chart populates it from Kueue's manager configuration unless an explicit populator selector is configured. + +Example: ```yaml - containers: - - args: - - "--zap-log-level=2" - - "--local-queue-name=my-custom-queue" +localQueueName: my-custom-queue +managedJobsNamespaceSelector: + matchLabels: + team: ml ``` ## Testing diff --git a/cmd/experimental/kueue-populator/charts/kueue-populator/README.md b/cmd/experimental/kueue-populator/charts/kueue-populator/README.md index 707847ef255..66b683b2a60 100644 --- a/cmd/experimental/kueue-populator/charts/kueue-populator/README.md +++ b/cmd/experimental/kueue-populator/charts/kueue-populator/README.md @@ -8,7 +8,7 @@ This Helm chart installs the Kueue Populator, a component designed to automatica - Installs Kueue (via subchart dependency). - Creates a default `ResourceFlavor` named `tas-gpu-default`. - Creates a default `ClusterQueue` (name configurable). -- The populator then creates a default `LocalQueue` (name configurable) in namespaces matching the selector, pointing to the default ClusterQueue. +- The populator then creates a default `LocalQueue` (name configurable) in namespaces matching Kueue's `managedJobsNamespaceSelector`, pointing to the default ClusterQueue. ## Prerequisites @@ -149,7 +149,8 @@ The following table lists the configurable parameters under the `kueuePopulator` | `config.clusterQueue.resources` | list | (see values.yaml) | Resources to configure in the default ResourceFlavor and ClusterQueue | | `config.topology.levels` | list | `[]` | Optional list of node labels for Topology Aware Scheduling levels. Enables Topology creation. | | `config.resourceFlavor.nodeLabels` | object | `{}` | Node labels to associate with the default ResourceFlavor. | -| `config.managedJobsNamespaceSelector` | object | (see values.yaml) | Label selector to filter namespaces where the default LocalQueue will be created. Excludes system namespaces. | +| `config.managedJobsNamespaceSelector` | object | unset | Optional selector to override Kueue's `managedJobsNamespaceSelector` for the populator. | +| `kueue.managerConfig.controllerManagerConfigYaml` | string | (see Kueue chart) | Kueue controller manager configuration. The chart copies `managedJobsNamespaceSelector` from this config into the populator config. | ### Kueue Subchart Configuration @@ -157,7 +158,7 @@ This chart includes the official `kueue` chart as a dependency. You can configur - `kueue.enabled: false`: Disables the subchart installation by default. Set to `true` to install Kueue. - `kueue.controllerManager.featureGates`: Enables `TopologyAwareScheduling`. -- `kueue.managerConfig.controllerManagerConfigYaml`: Provides minimal necessary overrides for `apiVersion` and `managedJobsNamespaceSelector` to ensure compatibility and safe hook execution. +- `kueue.managerConfig.controllerManagerConfigYaml`: Provides Kueue's controller manager configuration. The chart copies its `managedJobsNamespaceSelector` into the populator config unless `kueuePopulator.config.managedJobsNamespaceSelector` is set explicitly. See the [Kueue chart README](https://github.com/kubernetes-sigs/kueue/blob/main/charts/kueue/README.md) for all possible Kueue configuration options. diff --git a/cmd/experimental/kueue-populator/charts/kueue-populator/templates/manager.yaml b/cmd/experimental/kueue-populator/charts/kueue-populator/templates/manager.yaml index 51ed3ba11ac..031dc46ace2 100644 --- a/cmd/experimental/kueue-populator/charts/kueue-populator/templates/manager.yaml +++ b/cmd/experimental/kueue-populator/charts/kueue-populator/templates/manager.yaml @@ -1,3 +1,24 @@ +{{- $kueueConfig := dict -}} +{{- with .Values.kueue }} +{{- with .managerConfig }} +{{- with .controllerManagerConfigYaml }} +{{- $kueueConfig = . | fromYaml -}} +{{- end }} +{{- end }} +{{- end }} +{{- $kueueNamespace := .Release.Namespace -}} +{{- with $kueueConfig.namespace }} +{{- $kueueNamespace = . -}} +{{- end }} +{{- $managedJobsNamespaceSelector := dict "matchExpressions" (list (dict "key" "kubernetes.io/metadata.name" "operator" "NotIn" "values" (list "kube-system" $kueueNamespace))) -}} +{{- if and (hasKey $kueueConfig "managedJobsNamespaceSelector") (ne $kueueConfig.managedJobsNamespaceSelector nil) }} +{{- $managedJobsNamespaceSelector = $kueueConfig.managedJobsNamespaceSelector -}} +{{- end }} +{{- with .Values.kueuePopulator.config }} +{{- if and (hasKey . "managedJobsNamespaceSelector") (ne .managedJobsNamespaceSelector nil) }} +{{- $managedJobsNamespaceSelector = .managedJobsNamespaceSelector -}} +{{- end }} +{{- end }} apiVersion: v1 kind: ServiceAccount metadata: @@ -67,10 +88,8 @@ data: {{- else }} localQueueName: {{ .Values.kueuePopulator.config.localQueue.name }} {{- end }} - {{- if .Values.kueuePopulator.config.managedJobsNamespaceSelector }} managedJobsNamespaceSelector: - {{- toYaml .Values.kueuePopulator.config.managedJobsNamespaceSelector | nindent 6 }} - {{- end }} +{{ $managedJobsNamespaceSelector | toYaml | indent 6 }} --- apiVersion: apps/v1 kind: Deployment diff --git a/cmd/experimental/kueue-populator/charts/kueue-populator/tests/manager_test.yaml b/cmd/experimental/kueue-populator/charts/kueue-populator/tests/manager_test.yaml index 06f8aaee1df..06179ef414e 100644 --- a/cmd/experimental/kueue-populator/charts/kueue-populator/tests/manager_test.yaml +++ b/cmd/experimental/kueue-populator/charts/kueue-populator/tests/manager_test.yaml @@ -30,20 +30,112 @@ tests: path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation value: false - - it: should configure ConfigMap correctly + - it: should configure ConfigMap with Kueue namespace selector documentIndex: 3 set: kueuePopulator.config.localQueue.name: "my-local-queue" + kueue.managerConfig.controllerManagerConfigYaml: | + apiVersion: config.kueue.x-k8s.io/v1beta2 + kind: Configuration + managedJobsNamespaceSelector: + matchLabels: + team: ml + asserts: + - matchRegex: + path: data["controller_manager_config.yaml"] + pattern: "localQueueName: my-local-queue" + - matchRegex: + path: data["controller_manager_config.yaml"] + pattern: | + managedJobsNamespaceSelector: + \s+matchLabels: + \s+team: ml + - notExists: + path: data["kueue_controller_manager_config.yaml"] + + - it: should let explicit populator namespace selector override Kueue selector + documentIndex: 3 + set: kueuePopulator.config.managedJobsNamespaceSelector: matchLabels: app: my-app + kueue.managerConfig.controllerManagerConfigYaml: | + apiVersion: config.kueue.x-k8s.io/v1beta2 + kind: Configuration + managedJobsNamespaceSelector: + matchLabels: + team: ml asserts: - matchRegex: path: data["controller_manager_config.yaml"] - pattern: "localQueueName: my-local-queue" + pattern: | + managedJobsNamespaceSelector: + \s+matchLabels: + \s+app: my-app + - notMatchRegex: + path: data["controller_manager_config.yaml"] + pattern: "team: ml" + + - it: should default ConfigMap namespace selector from release namespace + documentIndex: 3 + release: + namespace: custom-kueue-system + set: + kueue.managerConfig.controllerManagerConfigYaml: | + apiVersion: config.kueue.x-k8s.io/v1beta2 + kind: Configuration + asserts: - matchRegex: path: data["controller_manager_config.yaml"] - pattern: "managedJobsNamespaceSelector:\n\\s+matchExpressions:\n\\s+- key: kubernetes.io/metadata.name\n\\s+operator: NotIn\n\\s+values:\n\\s+- kube-system\n\\s+- kueue-system\n\\s+matchLabels:\n\\s+app: my-app" + pattern: | + managedJobsNamespaceSelector: + \s+matchExpressions: + \s+- key: kubernetes.io/metadata.name + \s+operator: NotIn + \s+values: + \s+- kube-system + \s+- custom-kueue-system + + - it: should preserve explicit empty Kueue namespace selector + documentIndex: 3 + set: + kueue.managerConfig.controllerManagerConfigYaml: | + apiVersion: config.kueue.x-k8s.io/v1beta2 + kind: Configuration + managedJobsNamespaceSelector: {} + asserts: + - matchRegex: + path: data["controller_manager_config.yaml"] + pattern: | + managedJobsNamespaceSelector:\s+{} + - notMatchRegex: + path: data["controller_manager_config.yaml"] + pattern: "kube-system" + + - it: should default null Kueue namespace selector from release namespace + documentIndex: 3 + release: + namespace: custom-kueue-system + set: + kueue.managerConfig.controllerManagerConfigYaml: | + apiVersion: config.kueue.x-k8s.io/v1beta2 + kind: Configuration + managedJobsNamespaceSelector: null + asserts: + - matchRegex: + path: data["controller_manager_config.yaml"] + pattern: | + managedJobsNamespaceSelector: + \s+matchExpressions: + \s+- key: kubernetes.io/metadata.name + \s+operator: NotIn + \s+values: + \s+- kube-system + \s+- custom-kueue-system + - notMatchRegex: + path: data["controller_manager_config.yaml"] + pattern: | + managedJobsNamespaceSelector:\s+null - it: should set imagePullSecrets documentIndex: 4 diff --git a/cmd/experimental/kueue-populator/charts/kueue-populator/values.yaml b/cmd/experimental/kueue-populator/charts/kueue-populator/values.yaml index 7d22b8dfc32..eb1dce1e892 100644 --- a/cmd/experimental/kueue-populator/charts/kueue-populator/values.yaml +++ b/cmd/experimental/kueue-populator/charts/kueue-populator/values.yaml @@ -48,12 +48,9 @@ kueuePopulator: # nodeLabels: # cloud.google.com/gke-nodepool: "default-pool" nodeLabels: {} - # -- Label selector to filter namespaces where the default LocalQueue will be created. - managedJobsNamespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: ["kube-system", "kueue-system"] + # -- Optional selector to override Kueue's managedJobsNamespaceSelector for the populator. + # If unset, the chart copies Kueue's managedJobsNamespaceSelector from the Kueue manager configuration. + # managedJobsNamespaceSelector: {} # Default values for the dependency 'kueue' kueue: # -- Enable the Kueue subchart