From f39d88d4852570731a30ec1c363102cdde9ae557 Mon Sep 17 00:00:00 2001 From: Guillaume Carre Date: Thu, 30 Apr 2026 18:04:47 +0200 Subject: [PATCH 1/7] security: disable anonymous auth on kube-apiserver (MK8S-187) kube-apiserver was running with the default --anonymous-auth=true, letting unauthenticated callers reach endpoints bound to system:public-info-viewer (e.g. /version, /healthz) and disclose Kubernetes/Go versions usable for further attack planning. Set --anonymous-auth=false so the system:anonymous user cannot authenticate at all, which neutralises the default kubeadm-managed ClusterRoleBindings granting access to system:unauthenticated. Signed-off-by: Guillaume Carre --- salt/metalk8s/kubernetes/apiserver/installed.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/metalk8s/kubernetes/apiserver/installed.sls b/salt/metalk8s/kubernetes/apiserver/installed.sls index a9a0afc368..c89313e9bb 100644 --- a/salt/metalk8s/kubernetes/apiserver/installed.sls +++ b/salt/metalk8s/kubernetes/apiserver/installed.sls @@ -85,6 +85,7 @@ Create kube-apiserver Pod manifest: - kube-apiserver - --advertise-address={{ host }} - --allow-privileged=true + - --anonymous-auth=false - --authorization-mode=Node,RBAC - --client-ca-file=/etc/kubernetes/pki/ca.crt - --enable-admission-plugins=NodeRestriction From 8517e33b50629d53b101ebf96da6101800180e51 Mon Sep 17 00:00:00 2001 From: Guillaume Carre Date: Mon, 4 May 2026 08:53:24 +0200 Subject: [PATCH 2/7] tests: assert kube-apiserver rejects anonymous requests (MK8S-187) Bake the security advisory's diagnostic into the post-deploy BDD suite so a regression on --anonymous-auth is caught by CI: hit the advisory's exact endpoint (/api/kubernetes/version through the control-plane ingress, which proxies to kube-apiserver) without credentials and expect a 401 Unauthorized. Reuses existing 'perform a request on ... on control-plane Ingress' and 'the server returns ... with message ...' steps; no new step code. Signed-off-by: Guillaume Carre --- tests/post/features/authentication.feature | 5 +++++ tests/post/steps/test_authentication.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/tests/post/features/authentication.feature b/tests/post/features/authentication.feature index 699ca3b4b1..b5fa4e4a07 100644 --- a/tests/post/features/authentication.feature +++ b/tests/post/features/authentication.feature @@ -21,6 +21,11 @@ Feature: Authentication is up and running When we perform a request on '/oidc/' on control-plane Ingress Then the server returns '404' with message '404 page not found' + Scenario: kube-apiserver rejects anonymous requests + Given the Kubernetes API is available + When we perform a request on '/api/kubernetes/version' on control-plane Ingress + Then the server returns '401' with message 'Unauthorized' + Scenario: Login to Dex using incorrect email Given the Kubernetes API is available And the control-plane Ingress path '/oidc' is available diff --git a/tests/post/steps/test_authentication.py b/tests/post/steps/test_authentication.py index 97c4ca556e..f25896f2cf 100644 --- a/tests/post/steps/test_authentication.py +++ b/tests/post/steps/test_authentication.py @@ -36,6 +36,14 @@ def test_access_https_service(host): pass +@scenario( + "../features/authentication.feature", + "kube-apiserver rejects anonymous requests", +) +def test_apiserver_rejects_anonymous(host): + pass + + @scenario("../features/authentication.feature", "Login to Dex using incorrect email") def test_failed_login(host): pass From 68189f9f3046eee52c702b53c143bd666c062e57 Mon Sep 17 00:00:00 2001 From: Guillaume Carre Date: Tue, 5 May 2026 17:14:39 +0200 Subject: [PATCH 3/7] fix: authenticate kube-apiserver health probes on healthz endpoint (MK8S-187) Disabling --anonymous-auth in commit f39d88d4 caused bootstrap to hang on the salt http.wait_for_successful_query probes that hit /healthz on kube-apiserver: the unauthenticated GET now returns 401, the state times out after 5 minutes, and every dependent orchestration step fails with "One or more requisite failed". Pass the apiserver-kubelet client cert and key (CN kube-apiserver-kubelet-client, O system:masters) to all seven affected probes -- the direct one on https://:6443/healthz in kubernetes/apiserver/installed.sls, and the six going through the local apiserver-proxy on https://127.0.0.1:7443/healthz in the bootstrap, deploy_node, upgrade and downgrade orchestrations. The salt-master Pod already mounts /etc/kubernetes/pki, and the cert exists on every master before kube-apiserver starts, so no extra plumbing is needed. Expose apiserver-kubelet-client.key as certificates.client.files['apiserver-kubelet'].key in defaults.yaml and switch existing literal references in apiserver/installed.sls and apiserver/certs/kubelet-client.sls to it, to keep the path in a single place. Signed-off-by: Guillaume Carre --- salt/metalk8s/defaults.yaml | 1 + .../metalk8s/kubernetes/apiserver/certs/kubelet-client.sls | 2 +- salt/metalk8s/kubernetes/apiserver/installed.sls | 7 +++++-- salt/metalk8s/orchestrate/bootstrap/init.sls | 4 ++++ salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls | 4 ++++ salt/metalk8s/orchestrate/bootstrap/pre-upgrade.sls | 5 +++++ salt/metalk8s/orchestrate/deploy_node.sls | 7 +++++++ salt/metalk8s/orchestrate/downgrade/init.sls | 4 ++++ salt/metalk8s/orchestrate/upgrade/init.sls | 5 ++++- 9 files changed, 35 insertions(+), 4 deletions(-) diff --git a/salt/metalk8s/defaults.yaml b/salt/metalk8s/defaults.yaml index 0bd24522bf..f7ff77793d 100644 --- a/salt/metalk8s/defaults.yaml +++ b/salt/metalk8s/defaults.yaml @@ -223,6 +223,7 @@ certificates: watched: False apiserver-kubelet: path: /etc/kubernetes/pki/apiserver-kubelet-client.crt + key: /etc/kubernetes/pki/apiserver-kubelet-client.key renew: sls: - metalk8s.kubernetes.apiserver.certs.kubelet-client diff --git a/salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls b/salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls index d0ce6febf8..4204e8ae2e 100644 --- a/salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls +++ b/salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls @@ -1,7 +1,7 @@ {%- from "metalk8s/map.jinja" import certificates with context %} {%- from "metalk8s/map.jinja" import kube_api with context %} -{%- set private_key_path = "/etc/kubernetes/pki/apiserver-kubelet-client.key" %} +{%- set private_key_path = certificates.client.files['apiserver-kubelet'].key %} include: - metalk8s.internal.m2crypto diff --git a/salt/metalk8s/kubernetes/apiserver/installed.sls b/salt/metalk8s/kubernetes/apiserver/installed.sls index c89313e9bb..1581fee7af 100644 --- a/salt/metalk8s/kubernetes/apiserver/installed.sls +++ b/salt/metalk8s/kubernetes/apiserver/installed.sls @@ -63,7 +63,7 @@ Create kube-apiserver Pod manifest: - {{ certificates.client.files['apiserver-etcd'].path }} - /etc/kubernetes/pki/apiserver-etcd-client.key - {{ certificates.client.files['apiserver-kubelet'].path }} - - /etc/kubernetes/pki/apiserver-kubelet-client.key + - {{ certificates.client.files['apiserver-kubelet'].key }} - /etc/kubernetes/pki/ca.crt - /etc/kubernetes/pki/etcd/ca.crt - /etc/kubernetes/pki/front-proxy-ca.crt @@ -95,7 +95,7 @@ Create kube-apiserver Pod manifest: - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key - --etcd-servers={{ etcd_servers | join(",") }} - --kubelet-client-certificate={{ certificates.client.files['apiserver-kubelet'].path }} - - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key + - --kubelet-client-key={{ certificates.client.files['apiserver-kubelet'].key }} - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --proxy-client-cert-file={{ certificates.client.files['front-proxy'].path }} - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key @@ -193,6 +193,9 @@ Make sure kube-apiserver container is up and ready: - name: https://{{ host }}:6443/healthz - verify_ssl: True - ca_bundle: /etc/kubernetes/pki/ca.crt + - cert: + - {{ certificates.client.files['apiserver-kubelet'].path }} + - {{ certificates.client.files['apiserver-kubelet'].key }} - status: 200 - match: 'ok' - request_interval: 1 diff --git a/salt/metalk8s/orchestrate/bootstrap/init.sls b/salt/metalk8s/orchestrate/bootstrap/init.sls index f53ee8d411..4773b1fa27 100644 --- a/salt/metalk8s/orchestrate/bootstrap/init.sls +++ b/salt/metalk8s/orchestrate/bootstrap/init.sls @@ -1,5 +1,6 @@ {# Because of the grain lookup below, the bootstrap minion *must* be available before invoking this SLS, otherwise rendering will fail #} +{%- from "metalk8s/map.jinja" import certificates with context %} {%- set max_try = 5 %} {%- set bootstrap_grains = {} %} @@ -123,6 +124,9 @@ Wait for API server to be available: - match: 'ok' - status: 200 - verify_ssl: false + - cert: + - {{ certificates.client.files['apiserver-kubelet'].path }} + - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Bring bootstrap minion to highstate diff --git a/salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls b/salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls index 03a2400563..e939191547 100644 --- a/salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls +++ b/salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls @@ -1,3 +1,4 @@ +{%- from "metalk8s/map.jinja" import certificates with context %} {%- set dest_version = pillar.metalk8s.cluster_version %} Refresh CA minion: @@ -26,6 +27,9 @@ Wait for an API server to be available through local proxy: - match: 'ok' - status: 200 - verify_ssl: false + - cert: + - {{ certificates.client.files['apiserver-kubelet'].path }} + - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Prepare for Salt Master downgrade diff --git a/salt/metalk8s/orchestrate/bootstrap/pre-upgrade.sls b/salt/metalk8s/orchestrate/bootstrap/pre-upgrade.sls index 434acc7395..b8604f5f3d 100644 --- a/salt/metalk8s/orchestrate/bootstrap/pre-upgrade.sls +++ b/salt/metalk8s/orchestrate/bootstrap/pre-upgrade.sls @@ -1,3 +1,5 @@ +{%- from "metalk8s/map.jinja" import certificates with context %} + Refresh CA minion: salt.state: - sls: @@ -25,6 +27,9 @@ Wait for an API server to be available through local proxy: - match: 'ok' - status: 200 - verify_ssl: false + - cert: + - {{ certificates.client.files['apiserver-kubelet'].path }} + - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Prepare for Salt Master upgrade diff --git a/salt/metalk8s/orchestrate/deploy_node.sls b/salt/metalk8s/orchestrate/deploy_node.sls index 6f13316367..83692ec5d2 100644 --- a/salt/metalk8s/orchestrate/deploy_node.sls +++ b/salt/metalk8s/orchestrate/deploy_node.sls @@ -2,6 +2,7 @@ {%- from "metalk8s/map.jinja" import kubernetes with context %} {%- from "metalk8s/map.jinja" import networks with context %} {%- from "metalk8s/map.jinja" import repo with context %} +{%- from "metalk8s/map.jinja" import certificates with context %} {%- set node_name = pillar.orchestrate.node_name %} {%- set run_drain = not pillar.orchestrate.get('skip_draining', False) %} @@ -262,6 +263,9 @@ Wait for API server to be available before highstate: - match: 'ok' - status: 200 - verify_ssl: false + - cert: + - {{ certificates.client.files['apiserver-kubelet'].path }} + - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 Check pillar before highstate: @@ -308,6 +312,9 @@ Wait for API server to be available: - match: 'ok' - status: 200 - verify_ssl: false + - cert: + - {{ certificates.client.files['apiserver-kubelet'].path }} + - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 Uncordon the node: diff --git a/salt/metalk8s/orchestrate/downgrade/init.sls b/salt/metalk8s/orchestrate/downgrade/init.sls index ee6cd01699..c42a78a4eb 100644 --- a/salt/metalk8s/orchestrate/downgrade/init.sls +++ b/salt/metalk8s/orchestrate/downgrade/init.sls @@ -1,3 +1,4 @@ +{%- from "metalk8s/map.jinja" import certificates with context %} {%- set dest_version = pillar.metalk8s.cluster_version %} {#- NOTE: This orchestrate is called with a `salt-master` running the `dest_version` so this orchestrate need to be backward compatible. #} @@ -34,6 +35,9 @@ Wait for API server to be available on {{ node }}: - match: 'ok' - status: 200 - verify_ssl: false + - cert: + - {{ certificates.client.files['apiserver-kubelet'].path }} + - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Execute the downgrade prechecks diff --git a/salt/metalk8s/orchestrate/upgrade/init.sls b/salt/metalk8s/orchestrate/upgrade/init.sls index b8892e2923..d18ed0dcd8 100644 --- a/salt/metalk8s/orchestrate/upgrade/init.sls +++ b/salt/metalk8s/orchestrate/upgrade/init.sls @@ -2,7 +2,7 @@ # instead upgrades nodes fully (highstate), one by one. # This orchestrate should only be called after several other upgrade # steps, refer to the upgrade script. - +{%- from "metalk8s/map.jinja" import certificates with context %} {%- set dest_version = pillar.metalk8s.cluster_version %} Execute the upgrade prechecks: @@ -62,6 +62,9 @@ Wait for API server to be available on {{ node }}: - match: 'ok' - status: 200 - verify_ssl: false + - cert: + - {{ certificates.client.files['apiserver-kubelet'].path }} + - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Install apiserver-proxy on {{ node }} From 250a7d054ec1db62fddd383f76a38a08551e84a7 Mon Sep 17 00:00:00 2001 From: Guillaume Carre Date: Tue, 5 May 2026 17:45:50 +0200 Subject: [PATCH 4/7] security: scope kube-apiserver anonymous auth to health endpoints (MK8S-187) Switch kube-apiserver from --anonymous-auth=false to an AuthenticationConfiguration that allows anonymous access to /livez, /readyz and /healthz only. This restores kubelet livenessProbe, startupProbe and readinessProbe (plain httpGet, no way to attach credentials) while every other endpoint -- /version, /api/*, the discovery surface -- still requires authentication, so the original goal of MK8S-187 (neutralising the kubeadm system:public-info-viewer binding to system:anonymous) is preserved. The previous --anonymous-auth=false approach broke kubelet probes: unauthenticated GETs to /livez and /readyz returned 401, and after startupProbe.failureThreshold * periodSeconds (~250s) kubelet would have killed and restarted the apiserver in a permanent crash-loop. The salt http.wait_for_successful_query timeout on /healthz (fixed in 68189f9f3 by attaching a client cert) was the visible symptom; the kubelet-probe failure is the underlying one and cannot be fixed with client certs because httpGet probes do not support TLS client auth. Generate /etc/kubernetes/authentication-config.yaml from a sibling salt state (mirroring cryptconfig.sls), mount it into the static pod as a File volume, and wire --authentication-config to it. --anonymous-auth is dropped because it is mutually exclusive with the anonymous block in AuthenticationConfiguration. Move the OIDC issuer configuration from --oidc-* CLI flags to the AuthenticationConfiguration `jwt:` array. K8s 1.32 rejects the two mechanisms together (pkg/kubeapiserver/options/authentication.go: "authentication-config file and oidc-* flags are mutually exclusive"); a first apiserver start can succeed before the control-plane Ingress endpoint is known (no --oidc-* flags emitted), but the subsequent "Reconfigure control plane Ingress" pass populates oidc_config in the pillar, re-renders the manifest with both flag sets, and the apiserver crash-loops on startup. The same five fields the legacy code consumed (issuerURL, clientID, CAFile, usernameClaim, groupsClaim) feed the new jwt issuer block. The CA PEM is read from the salt mine entry `ingress_ca_b64` for the default Dex / Ingress path (avoiding any state-graph ordering hazard against the on-disk file), and falls back to salt['file.read'] for a pillar-supplied OIDC override pointing at a different CA path. When the username claim is `email`, a `claimValidationRules` CEL expression `claims.?email_verified.orValue(true)` is added to reproduce the implicit `email_verified == true` guard that the legacy --oidc-username-claim=email flag used to apply automatically. ARTESCA configures Keycloak via the same OIDC pillar override, so its reconfiguration path is unaffected. Relies on the AnonymousAuthConfigurableEndpoints and StructuredAuthenticationConfiguration feature gates, both beta and on-by-default in Kubernetes 1.32 (per pkg/features/versioned_kube_features.go in release-1.32) -- no extra --feature-gates flag needed for the version metalk8s pins. Extend the post-install authentication feature with two scenario outlines that lock the contract in place: anonymous access to /livez, /readyz, /healthz returns 200 'ok' (via the control-plane Ingress, same path as the existing "rejects anonymous" test), and authenticated access (admin client cert from the kubeconfig) to the same three paths also returns 200 'ok'. The pre-existing /version 401 scenario still passes because /version is not in the allowed anonymous path list. Signed-off-by: Guillaume Carre --- buildchain/buildchain/salt_tree.py | 1 + .../kubernetes/apiserver/authnconfig.sls | 113 ++++++++++++++++++ salt/metalk8s/kubernetes/apiserver/init.sls | 2 + .../kubernetes/apiserver/installed.sls | 31 ++--- tests/post/features/authentication.feature | 22 ++++ tests/post/steps/test_authentication.py | 100 +++++++++++++++- 6 files changed, 245 insertions(+), 24 deletions(-) create mode 100644 salt/metalk8s/kubernetes/apiserver/authnconfig.sls diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index 7fd198c18f..6de56b5e04 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -522,6 +522,7 @@ def _download_ui_operator_crds() -> str: Path("salt/metalk8s/kubernetes/apiserver/certs/server.sls"), Path("salt/metalk8s/kubernetes/apiserver/init.sls"), Path("salt/metalk8s/kubernetes/apiserver/installed.sls"), + Path("salt/metalk8s/kubernetes/apiserver/authnconfig.sls"), Path("salt/metalk8s/kubernetes/apiserver/cryptconfig.sls"), Path("salt/metalk8s/kubernetes/apiserver/kubeconfig.sls"), Path("salt/metalk8s/kubernetes/apiserver-proxy/files/apiserver-proxy.conf.j2"), diff --git a/salt/metalk8s/kubernetes/apiserver/authnconfig.sls b/salt/metalk8s/kubernetes/apiserver/authnconfig.sls new file mode 100644 index 0000000000..ae0ca73af5 --- /dev/null +++ b/salt/metalk8s/kubernetes/apiserver/authnconfig.sls @@ -0,0 +1,113 @@ +# Generate the AuthenticationConfiguration file consumed by kube-apiserver +# via --authentication-config. +# +# It carries: +# * `anonymous` -- limits anonymous access to the kubelet probe endpoints +# (/livez, /readyz, /healthz) so kubelet httpGet probes keep working, +# while every other path (/version, /api/*, ...) still requires +# authentication. +# * `jwt` -- the OIDC issuer (Dex by default, or a pillar override). This +# replaces the legacy --oidc-* command-line flags, which are mutually +# exclusive with --authentication-config in Kubernetes 1.32+ +# (pkg/kubeapiserver/options/authentication.go: "authentication-config +# file and oidc-* flags are mutually exclusive"). +# +# Relies on the AnonymousAuthConfigurableEndpoints and +# StructuredAuthenticationConfiguration feature gates, both beta and +# on-by-default in Kubernetes 1.32+. + +include: + - .installed + - metalk8s.addons.nginx-ingress.ca.advertised + +{%- set authn_config_path = '/etc/kubernetes/authentication-config.yaml' %} + +{#- Build the OIDC issuer config, mirroring the historical --oidc-* selection + logic that used to live in installed.sls. #} +{%- set oidc_config = {} %} +{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %} + {%- do oidc_config.update(pillar.kubernetes.apiServer.oidc) %} +{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %} + {%- do oidc_config.update({ + "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc", + "clientID": "oidc-auth-client", + "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt", + "usernameClaim": "email", + "groupsClaim": "groups", + }) %} +{%- endif %} + +{#- AuthenticationConfiguration's `jwt[].issuer.certificateAuthority` field + expects PEM content inline, not a file path. For the default Dex case the + Ingress CA is published in the salt mine as `ingress_ca_b64` by + `metalk8s.addons.nginx-ingress.ca.installed`, which avoids any ordering + dependency on the on-disk file. For a pillar-provided OIDC override we + fall back to reading the user-specified CAFile from the salt master. #} +{%- set ca_pem = '' %} +{%- if oidc_config %} +{%- set ingress_ca_path = '/etc/metalk8s/pki/nginx-ingress/ca.crt' %} +{%- if oidc_config.get('CAFile') == ingress_ca_path %} +{%- set ingress_ca_mine = salt['mine.get'](pillar.metalk8s.ca.minion, 'ingress_ca_b64') %} +{%- if ingress_ca_mine %} +{%- set ca_pem = salt['hashutil.base64_b64decode'](ingress_ca_mine[pillar.metalk8s.ca.minion]) %} +{%- endif %} +{%- elif oidc_config.get('CAFile') %} +{%- set ca_pem = salt['file.read'](oidc_config.CAFile) %} +{%- endif %} +{%- endif %} + +{#- TODO(MK8S-258): bump apiVersion to apiserver.config.k8s.io/v1 once metalk8s + pins Kubernetes >= 1.34. AuthenticationConfiguration is registered in + v1beta1 in 1.32/1.33 and promotes to v1 (GA) in 1.34. #} +{%- set authn_config = { + "apiVersion": "apiserver.config.k8s.io/v1beta1", + "kind": "AuthenticationConfiguration", + "anonymous": { + "enabled": True, + "conditions": [ + {"path": "/livez"}, + {"path": "/readyz"}, + {"path": "/healthz"}, + ], + }, +} %} + +{%- if oidc_config and ca_pem %} +{%- set jwt_authenticator = { + "issuer": { + "url": oidc_config.issuerURL, + "audiences": [oidc_config.clientID], + "certificateAuthority": ca_pem, + }, + "claimMappings": { + "username": {"claim": oidc_config.usernameClaim, "prefix": "oidc:"}, + "groups": {"claim": oidc_config.groupsClaim, "prefix": "oidc:"}, + }, +} %} +{#- Reproduce the legacy --oidc-username-claim=email implicit guard: when the + username claim is `email`, kube-apiserver used to auto-require + `email_verified == true`. With AuthenticationConfiguration, we have to + spell the rule out as a CEL expression. #} +{%- if oidc_config.usernameClaim == 'email' %} +{%- do jwt_authenticator.update({ + "claimValidationRules": [ + { + "expression": "claims.?email_verified.orValue(true) == true", + "message": "email_verified claim must be true when set", + }, + ], +}) %} +{%- endif %} +{%- do authn_config.update({"jwt": [jwt_authenticator]}) %} +{%- endif %} + +Create kube-apiserver authentication configuration: + file.serialize: + - name: {{ authn_config_path }} + - mode: '0600' + - user: root + - group: root + - makedirs: True + - dataset: {{ authn_config | tojson }} + - require_in: + - metalk8s: Create kube-apiserver Pod manifest diff --git a/salt/metalk8s/kubernetes/apiserver/init.sls b/salt/metalk8s/kubernetes/apiserver/init.sls index d47b3f8850..68040b055e 100644 --- a/salt/metalk8s/kubernetes/apiserver/init.sls +++ b/salt/metalk8s/kubernetes/apiserver/init.sls @@ -7,8 +7,10 @@ # * installed -> deploy apiserver manifest # * kubeconfig -> create admin kubeconfig file # * cryptconfig -> create apiserver encryption configuration +# * authnconfig -> create apiserver authentication configuration # include: - .installed - .kubeconfig - .cryptconfig + - .authnconfig diff --git a/salt/metalk8s/kubernetes/apiserver/installed.sls b/salt/metalk8s/kubernetes/apiserver/installed.sls index 1581fee7af..28298fbaf4 100644 --- a/salt/metalk8s/kubernetes/apiserver/installed.sls +++ b/salt/metalk8s/kubernetes/apiserver/installed.sls @@ -5,6 +5,7 @@ {%- from "metalk8s/map.jinja" import networks with context %} {%- set encryption_k8s_path = "/etc/kubernetes/encryption.conf" %} +{%- set authn_config_path = "/etc/kubernetes/authentication-config.yaml" %} include: - metalk8s.kubernetes.ca.advertised @@ -34,18 +35,9 @@ include: {%- do feature_gates.append(feature ~ "=" ~ value) %} {%- endfor %} -{%- set oidc_config = {} %} -{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %} - {%- do oidc_config.update(pillar.kubernetes.apiServer.oidc) %} -{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %} - {%- do oidc_config.update({ - "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc", - "clientID": "oidc-auth-client", - "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt", - "usernameClaim": "email", - "groupsClaim": "groups", - }) %} -{%- endif %} +{# OIDC is configured via the AuthenticationConfiguration file written by + .authnconfig (--oidc-* flags are mutually exclusive with + --authentication-config in Kubernetes 1.32+). #} {%- set pod_name = "kube-apiserver-" ~ grains.id %} {%- set last_pod_id = salt.cri.get_pod_id( @@ -58,6 +50,7 @@ Create kube-apiserver Pod manifest: - source: salt://metalk8s/kubernetes/files/control-plane-manifest.yaml.j2 - config_files: - {{ encryption_k8s_path }} + - {{ authn_config_path }} - {{ certificates.server.files.apiserver.path }} - /etc/kubernetes/pki/apiserver.key - {{ certificates.client.files['apiserver-etcd'].path }} @@ -85,7 +78,7 @@ Create kube-apiserver Pod manifest: - kube-apiserver - --advertise-address={{ host }} - --allow-privileged=true - - --anonymous-auth=false + - --authentication-config={{ authn_config_path }} - --authorization-mode=Node,RBAC - --client-ca-file=/etc/kubernetes/pki/ca.crt - --enable-admission-plugins=NodeRestriction @@ -115,15 +108,6 @@ Create kube-apiserver Pod manifest: - --bind-address={{ host }} - --encryption-provider-config={{ encryption_k8s_path }} - --cors-allowed-origins=^.*$ - {%- if oidc_config %} - - --oidc-issuer-url={{ oidc_config.issuerURL }} - - --oidc-client-id={{ oidc_config.clientID }} - - --oidc-ca-file={{ oidc_config.CAFile }} - - --oidc-username-claim={{ oidc_config.usernameClaim }} - - --oidc-groups-claim={{ oidc_config.groupsClaim }} - - '"--oidc-username-prefix=oidc:"' - - '"--oidc-groups-prefix=oidc:"' - {%- endif %} - --v={{ 2 if metalk8s.debug else 0 }} {% if feature_gates %} - --feature-gates={{ feature_gates | join(",") }} @@ -133,6 +117,9 @@ Create kube-apiserver Pod manifest: - path: {{ encryption_k8s_path }} type: File name: k8s-encryption + - path: {{ authn_config_path }} + type: File + name: k8s-authn-config {%- if grains['os_family'] == 'RedHat' %} - path: /etc/pki/ca-trust name: etc-pki-ca-trust diff --git a/tests/post/features/authentication.feature b/tests/post/features/authentication.feature index b5fa4e4a07..ee82a22ab8 100644 --- a/tests/post/features/authentication.feature +++ b/tests/post/features/authentication.feature @@ -26,6 +26,28 @@ Feature: Authentication is up and running When we perform a request on '/api/kubernetes/version' on control-plane Ingress Then the server returns '401' with message 'Unauthorized' + Scenario Outline: kube-apiserver allows anonymous access to + Given the Kubernetes API is available + When we perform an anonymous request on the API server '' endpoint + Then the server returns '200' with message 'ok' + + Examples: + | path | + | livez | + | readyz | + | healthz | + + Scenario Outline: kube-apiserver accepts authenticated access to + Given the Kubernetes API is available + When we perform an authenticated request on the API server '' endpoint + Then the server returns '200' with message 'ok' + + Examples: + | path | + | livez | + | readyz | + | healthz | + Scenario: Login to Dex using incorrect email Given the Kubernetes API is available And the control-plane Ingress path '/oidc' is available diff --git a/tests/post/steps/test_authentication.py b/tests/post/steps/test_authentication.py index f25896f2cf..b43b1e2d30 100644 --- a/tests/post/steps/test_authentication.py +++ b/tests/post/steps/test_authentication.py @@ -44,6 +44,24 @@ def test_apiserver_rejects_anonymous(host): pass +@scenario( + "../features/authentication.feature", + "kube-apiserver allows anonymous access to ", + example_converters={"path": str}, +) +def test_apiserver_allows_anonymous_health(host): + pass + + +@scenario( + "../features/authentication.feature", + "kube-apiserver accepts authenticated access to ", + example_converters={"path": str}, +) +def test_apiserver_accepts_authenticated_health(host): + pass + + @scenario("../features/authentication.feature", "Login to Dex using incorrect email") def test_failed_login(host): pass @@ -117,6 +135,46 @@ def perform_request(host, context, control_plane_ingress_ep, path): pytest.fail("Failed to access oidc url path with error: {}".format(exc)) +@when("we perform an anonymous request on the API server '' endpoint") +def perform_anonymous_apiserver_request(context, control_plane_ip, path): + """Hit kube-apiserver directly on :6443 with no credentials. + + Bypasses the control-plane Ingress so the path the apiserver sees is + exactly `/`, with no rewrite ambiguity. `path` is the example + value (e.g. "livez") supplied by the Scenario Outline. We use the + literal-text decorator style here -- in pytest-bdd 3.2.1 only this + style substitutes `` references; parsers.parse(...) keeps + them literal. + """ + session = utils.requests_retry_session() + try: + context["response"] = session.get( + "https://{ip}:6443/{path}".format(ip=control_plane_ip, path=path), + verify=False, + ) + except requests.exceptions.ConnectionError as exc: + pytest.fail("Failed to access API server with error: {}".format(exc)) + + +@when("we perform an authenticated request on the API server '' endpoint") +def perform_authenticated_request(context, control_plane_ip, k8s_client, path): + """Hit kube-apiserver directly on :6443 with the admin client cert. + + Same notes as the anonymous variant on URL construction and on the + decorator style. + """ + config = k8s_client.client.configuration + session = utils.requests_retry_session() + try: + context["response"] = session.get( + "https://{ip}:6443/{path}".format(ip=control_plane_ip, path=path), + verify=config.ssl_ca_cert, + cert=(config.cert_file, config.key_file), + ) + except requests.exceptions.ConnectionError as exc: + pytest.fail("Failed to access API server with error: {}".format(exc)) + + # }}} # Then {{{ @@ -159,8 +217,46 @@ def _get_openID_config(): def server_returns(host, context, status_code, status_message): response = context.get("response") assert response is not None - assert response.status_code == int(status_code) - assert response.text.rstrip("\n") == status_message + + expected_code = int(status_code) + actual_url = response.request.url if response.request else "" + actual_body_excerpt = response.text[:500].replace("\n", "\\n") + + # kube-apiserver returns either: + # * a plain-text body (e.g. /livez returns "ok\n"), or + # * a structured `Status` JSON object (e.g. on a 401 with + # AuthenticationConfiguration in use, the body is + # {"kind":"Status",...,"message":"Unauthorized","code":401}). + # Compare against `message` for the JSON case, against the raw text + # otherwise. + try: + parsed = response.json() + except ValueError: + parsed = None + actual_message = ( + parsed.get("message") + if isinstance(parsed, dict) and parsed.get("kind") == "Status" + else response.text.rstrip("\n") + ) + + assert response.status_code == expected_code, ( + "Expected HTTP {expected} but got {actual} from {url}. " + "Body excerpt: {body}".format( + expected=expected_code, + actual=response.status_code, + url=actual_url, + body=actual_body_excerpt, + ) + ) + assert actual_message == status_message, ( + "Expected message {expected!r} but got {actual!r} from {url}. " + "Body excerpt: {body}".format( + expected=status_message, + actual=actual_message, + url=actual_url, + body=actual_body_excerpt, + ) + ) # }}} From 18e87ccf3bd7cd07e01c1b7ef9f42192513af647 Mon Sep 17 00:00:00 2001 From: Guillaume Carre Date: Mon, 18 May 2026 17:15:05 +0200 Subject: [PATCH 5/7] review: address code review on MK8S-187 kube-apiserver authn config * Centralize the authn config path in `kube_api.authn_config_path` (defaults.yaml) and consume it from both authnconfig.sls and installed.sls instead of duplicating the literal. * Inline the AuthenticationConfiguration dataset in authnconfig.sls as templated YAML rather than building a Jinja dict + tojson, for readability. * Drop the now-unused `cert:` (mTLS client cert) argument from the healthz http.wait_for_successful_query checks across the apiserver and orchestrate states, along with the matching `certificates` imports that became dead. --- salt/metalk8s/defaults.yaml | 1 + .../kubernetes/apiserver/authnconfig.sls | 81 ++++++++----------- .../kubernetes/apiserver/installed.sls | 9 +-- salt/metalk8s/orchestrate/bootstrap/init.sls | 4 - .../orchestrate/bootstrap/pre-downgrade.sls | 4 - .../orchestrate/bootstrap/pre-upgrade.sls | 5 -- salt/metalk8s/orchestrate/deploy_node.sls | 7 -- salt/metalk8s/orchestrate/downgrade/init.sls | 4 - salt/metalk8s/orchestrate/upgrade/init.sls | 5 +- 9 files changed, 40 insertions(+), 80 deletions(-) diff --git a/salt/metalk8s/defaults.yaml b/salt/metalk8s/defaults.yaml index f7ff77793d..8d99bd8081 100644 --- a/salt/metalk8s/defaults.yaml +++ b/salt/metalk8s/defaults.yaml @@ -150,6 +150,7 @@ kube_api: cert: server_signing_policy: kube_apiserver_server_policy client_signing_policy: kube_apiserver_client_policy + authn_config_path: /etc/kubernetes/authn_config.yaml etcd: ca: diff --git a/salt/metalk8s/kubernetes/apiserver/authnconfig.sls b/salt/metalk8s/kubernetes/apiserver/authnconfig.sls index ae0ca73af5..6f6f80b4d2 100644 --- a/salt/metalk8s/kubernetes/apiserver/authnconfig.sls +++ b/salt/metalk8s/kubernetes/apiserver/authnconfig.sls @@ -15,12 +15,13 @@ # Relies on the AnonymousAuthConfigurableEndpoints and # StructuredAuthenticationConfiguration feature gates, both beta and # on-by-default in Kubernetes 1.32+. +{%- from "metalk8s/map.jinja" import kube_api with context %} include: - .installed - metalk8s.addons.nginx-ingress.ca.advertised -{%- set authn_config_path = '/etc/kubernetes/authentication-config.yaml' %} +{%- set authn_config_path = kube_api.authn_config_path %} {#- Build the OIDC issuer config, mirroring the historical --oidc-* selection logic that used to live in installed.sls. #} @@ -56,50 +57,6 @@ include: {%- endif %} {%- endif %} -{#- TODO(MK8S-258): bump apiVersion to apiserver.config.k8s.io/v1 once metalk8s - pins Kubernetes >= 1.34. AuthenticationConfiguration is registered in - v1beta1 in 1.32/1.33 and promotes to v1 (GA) in 1.34. #} -{%- set authn_config = { - "apiVersion": "apiserver.config.k8s.io/v1beta1", - "kind": "AuthenticationConfiguration", - "anonymous": { - "enabled": True, - "conditions": [ - {"path": "/livez"}, - {"path": "/readyz"}, - {"path": "/healthz"}, - ], - }, -} %} - -{%- if oidc_config and ca_pem %} -{%- set jwt_authenticator = { - "issuer": { - "url": oidc_config.issuerURL, - "audiences": [oidc_config.clientID], - "certificateAuthority": ca_pem, - }, - "claimMappings": { - "username": {"claim": oidc_config.usernameClaim, "prefix": "oidc:"}, - "groups": {"claim": oidc_config.groupsClaim, "prefix": "oidc:"}, - }, -} %} -{#- Reproduce the legacy --oidc-username-claim=email implicit guard: when the - username claim is `email`, kube-apiserver used to auto-require - `email_verified == true`. With AuthenticationConfiguration, we have to - spell the rule out as a CEL expression. #} -{%- if oidc_config.usernameClaim == 'email' %} -{%- do jwt_authenticator.update({ - "claimValidationRules": [ - { - "expression": "claims.?email_verified.orValue(true) == true", - "message": "email_verified claim must be true when set", - }, - ], -}) %} -{%- endif %} -{%- do authn_config.update({"jwt": [jwt_authenticator]}) %} -{%- endif %} Create kube-apiserver authentication configuration: file.serialize: @@ -108,6 +65,38 @@ Create kube-apiserver authentication configuration: - user: root - group: root - makedirs: True - - dataset: {{ authn_config | tojson }} + - dataset: + {#- TODO(MK8S-258): bump apiVersion to apiserver.config.k8s.io/v1 once metalk8s + pins Kubernetes >= 1.34. AuthenticationConfiguration is registered in + v1beta1 in 1.32/1.33 and promotes to v1 (GA) in 1.34. + it will also continue to be supported in v1beta1 until 1.36 #} + apiVersion: apiserver.config.k8s.io/v1beta1 + kind: AuthenticationConfiguration + anonymous: + enabled: true + conditions: + - path: /livez + - path: /readyz + - path: /healthz + {%- if oidc_config and ca_pem %} + jwt: + - issuer: + url: {{ oidc_config.issuerURL }} + audiences: + - {{ oidc_config.clientID }} + certificateAuthority: {{ ca_pem }} + claimMappings: + username: + claim: {{ oidc_config.usernameClaim }} + prefix: "oidc:" + groups: + claim: {{ oidc_config.groupsClaim }} + prefix: "oidc:" + {%- if oidc_config.usernameClaim == 'email' %} + claimValidationRules: + - expression: "claims.?email_verified.orValue(true) == true" + message: "email_verified claim must be true when set" + {%- endif %} + {%- endif %} - require_in: - metalk8s: Create kube-apiserver Pod manifest diff --git a/salt/metalk8s/kubernetes/apiserver/installed.sls b/salt/metalk8s/kubernetes/apiserver/installed.sls index 28298fbaf4..5b8284184b 100644 --- a/salt/metalk8s/kubernetes/apiserver/installed.sls +++ b/salt/metalk8s/kubernetes/apiserver/installed.sls @@ -3,10 +3,10 @@ {%- from "metalk8s/map.jinja" import coredns with context %} {%- from "metalk8s/map.jinja" import metalk8s with context %} {%- from "metalk8s/map.jinja" import networks with context %} +{%- from "metalk8s/map.jinja" import kube_api with context %} {%- set encryption_k8s_path = "/etc/kubernetes/encryption.conf" %} -{%- set authn_config_path = "/etc/kubernetes/authentication-config.yaml" %} - +{%- set authn_config_path = kube_api.authn_config_path %} include: - metalk8s.kubernetes.ca.advertised - metalk8s.kubernetes.sa.advertised @@ -78,7 +78,6 @@ Create kube-apiserver Pod manifest: - kube-apiserver - --advertise-address={{ host }} - --allow-privileged=true - - --authentication-config={{ authn_config_path }} - --authorization-mode=Node,RBAC - --client-ca-file=/etc/kubernetes/pki/ca.crt - --enable-admission-plugins=NodeRestriction @@ -109,6 +108,7 @@ Create kube-apiserver Pod manifest: - --encryption-provider-config={{ encryption_k8s_path }} - --cors-allowed-origins=^.*$ - --v={{ 2 if metalk8s.debug else 0 }} + - --authentication-config={{ authn_config_path }} {% if feature_gates %} - --feature-gates={{ feature_gates | join(",") }} {%- endif %} @@ -180,9 +180,6 @@ Make sure kube-apiserver container is up and ready: - name: https://{{ host }}:6443/healthz - verify_ssl: True - ca_bundle: /etc/kubernetes/pki/ca.crt - - cert: - - {{ certificates.client.files['apiserver-kubelet'].path }} - - {{ certificates.client.files['apiserver-kubelet'].key }} - status: 200 - match: 'ok' - request_interval: 1 diff --git a/salt/metalk8s/orchestrate/bootstrap/init.sls b/salt/metalk8s/orchestrate/bootstrap/init.sls index 4773b1fa27..f53ee8d411 100644 --- a/salt/metalk8s/orchestrate/bootstrap/init.sls +++ b/salt/metalk8s/orchestrate/bootstrap/init.sls @@ -1,6 +1,5 @@ {# Because of the grain lookup below, the bootstrap minion *must* be available before invoking this SLS, otherwise rendering will fail #} -{%- from "metalk8s/map.jinja" import certificates with context %} {%- set max_try = 5 %} {%- set bootstrap_grains = {} %} @@ -124,9 +123,6 @@ Wait for API server to be available: - match: 'ok' - status: 200 - verify_ssl: false - - cert: - - {{ certificates.client.files['apiserver-kubelet'].path }} - - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Bring bootstrap minion to highstate diff --git a/salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls b/salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls index e939191547..03a2400563 100644 --- a/salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls +++ b/salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls @@ -1,4 +1,3 @@ -{%- from "metalk8s/map.jinja" import certificates with context %} {%- set dest_version = pillar.metalk8s.cluster_version %} Refresh CA minion: @@ -27,9 +26,6 @@ Wait for an API server to be available through local proxy: - match: 'ok' - status: 200 - verify_ssl: false - - cert: - - {{ certificates.client.files['apiserver-kubelet'].path }} - - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Prepare for Salt Master downgrade diff --git a/salt/metalk8s/orchestrate/bootstrap/pre-upgrade.sls b/salt/metalk8s/orchestrate/bootstrap/pre-upgrade.sls index b8604f5f3d..434acc7395 100644 --- a/salt/metalk8s/orchestrate/bootstrap/pre-upgrade.sls +++ b/salt/metalk8s/orchestrate/bootstrap/pre-upgrade.sls @@ -1,5 +1,3 @@ -{%- from "metalk8s/map.jinja" import certificates with context %} - Refresh CA minion: salt.state: - sls: @@ -27,9 +25,6 @@ Wait for an API server to be available through local proxy: - match: 'ok' - status: 200 - verify_ssl: false - - cert: - - {{ certificates.client.files['apiserver-kubelet'].path }} - - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Prepare for Salt Master upgrade diff --git a/salt/metalk8s/orchestrate/deploy_node.sls b/salt/metalk8s/orchestrate/deploy_node.sls index 83692ec5d2..6f13316367 100644 --- a/salt/metalk8s/orchestrate/deploy_node.sls +++ b/salt/metalk8s/orchestrate/deploy_node.sls @@ -2,7 +2,6 @@ {%- from "metalk8s/map.jinja" import kubernetes with context %} {%- from "metalk8s/map.jinja" import networks with context %} {%- from "metalk8s/map.jinja" import repo with context %} -{%- from "metalk8s/map.jinja" import certificates with context %} {%- set node_name = pillar.orchestrate.node_name %} {%- set run_drain = not pillar.orchestrate.get('skip_draining', False) %} @@ -263,9 +262,6 @@ Wait for API server to be available before highstate: - match: 'ok' - status: 200 - verify_ssl: false - - cert: - - {{ certificates.client.files['apiserver-kubelet'].path }} - - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 Check pillar before highstate: @@ -312,9 +308,6 @@ Wait for API server to be available: - match: 'ok' - status: 200 - verify_ssl: false - - cert: - - {{ certificates.client.files['apiserver-kubelet'].path }} - - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 Uncordon the node: diff --git a/salt/metalk8s/orchestrate/downgrade/init.sls b/salt/metalk8s/orchestrate/downgrade/init.sls index c42a78a4eb..ee6cd01699 100644 --- a/salt/metalk8s/orchestrate/downgrade/init.sls +++ b/salt/metalk8s/orchestrate/downgrade/init.sls @@ -1,4 +1,3 @@ -{%- from "metalk8s/map.jinja" import certificates with context %} {%- set dest_version = pillar.metalk8s.cluster_version %} {#- NOTE: This orchestrate is called with a `salt-master` running the `dest_version` so this orchestrate need to be backward compatible. #} @@ -35,9 +34,6 @@ Wait for API server to be available on {{ node }}: - match: 'ok' - status: 200 - verify_ssl: false - - cert: - - {{ certificates.client.files['apiserver-kubelet'].path }} - - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Execute the downgrade prechecks diff --git a/salt/metalk8s/orchestrate/upgrade/init.sls b/salt/metalk8s/orchestrate/upgrade/init.sls index d18ed0dcd8..b8892e2923 100644 --- a/salt/metalk8s/orchestrate/upgrade/init.sls +++ b/salt/metalk8s/orchestrate/upgrade/init.sls @@ -2,7 +2,7 @@ # instead upgrades nodes fully (highstate), one by one. # This orchestrate should only be called after several other upgrade # steps, refer to the upgrade script. -{%- from "metalk8s/map.jinja" import certificates with context %} + {%- set dest_version = pillar.metalk8s.cluster_version %} Execute the upgrade prechecks: @@ -62,9 +62,6 @@ Wait for API server to be available on {{ node }}: - match: 'ok' - status: 200 - verify_ssl: false - - cert: - - {{ certificates.client.files['apiserver-kubelet'].path }} - - {{ certificates.client.files['apiserver-kubelet'].key }} - request_interval: 1 - require: - salt: Install apiserver-proxy on {{ node }} From b3184f4852e6bf90ee9ca0741424c713e0565377 Mon Sep 17 00:00:00 2001 From: Guillaume Carre Date: Mon, 18 May 2026 18:09:11 +0200 Subject: [PATCH 6/7] review: added changelog entry Signed-off-by: Guillaume Carre --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8453812e81..c6ebab3e9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### Bug Fixes +- Disable anonymous authentication on `kube-apiserver`, except for the kubelet + probe endpoints (`livez`, `readyz`, `healthz`) + (PR[#4900](https://github.com/scality/metalk8s/pull/4900)) + - Fix a bug where Salt master process may report an error about `VerboseLogger` not having attributes `trace` (PR[#4831](https://github.com/scality/metalk8s/pull/4831)) From 0459545576fbfa169a1260a90d475c112278ca49 Mon Sep 17 00:00:00 2001 From: Guillaume Carre Date: Mon, 18 May 2026 19:41:27 +0200 Subject: [PATCH 7/7] review: Set ca_pem in same if/else code as for oidc_config (MK8S-187) Signed-off-by: Guillaume Carre --- .../kubernetes/apiserver/authnconfig.sls | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/salt/metalk8s/kubernetes/apiserver/authnconfig.sls b/salt/metalk8s/kubernetes/apiserver/authnconfig.sls index 6f6f80b4d2..92235b42ad 100644 --- a/salt/metalk8s/kubernetes/apiserver/authnconfig.sls +++ b/salt/metalk8s/kubernetes/apiserver/authnconfig.sls @@ -23,38 +23,35 @@ include: {%- set authn_config_path = kube_api.authn_config_path %} -{#- Build the OIDC issuer config, mirroring the historical --oidc-* selection - logic that used to live in installed.sls. #} +{#- Build the OIDC issuer config and resolve the matching CA PEM, mirroring the + historical --oidc-* selection logic that used to live in installed.sls. + AuthenticationConfiguration's `jwt[].issuer.certificateAuthority` field + expects PEM content inline (not a file path), so each branch resolves the + CA the way that fits its source: + * pillar override -- read the user-specified CAFile from the salt master. + * default Dex -- pull the Ingress CA from the salt mine + (`ingress_ca_b64`, published by + `metalk8s.addons.nginx-ingress.ca.installed`), which + avoids any ordering dependency on the on-disk file. #} {%- set oidc_config = {} %} -{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %} - {%- do oidc_config.update(pillar.kubernetes.apiServer.oidc) %} -{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %} - {%- do oidc_config.update({ - "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc", - "clientID": "oidc-auth-client", - "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt", - "usernameClaim": "email", - "groupsClaim": "groups", - }) %} -{%- endif %} - -{#- AuthenticationConfiguration's `jwt[].issuer.certificateAuthority` field - expects PEM content inline, not a file path. For the default Dex case the - Ingress CA is published in the salt mine as `ingress_ca_b64` by - `metalk8s.addons.nginx-ingress.ca.installed`, which avoids any ordering - dependency on the on-disk file. For a pillar-provided OIDC override we - fall back to reading the user-specified CAFile from the salt master. #} {%- set ca_pem = '' %} -{%- if oidc_config %} -{%- set ingress_ca_path = '/etc/metalk8s/pki/nginx-ingress/ca.crt' %} -{%- if oidc_config.get('CAFile') == ingress_ca_path %} -{%- set ingress_ca_mine = salt['mine.get'](pillar.metalk8s.ca.minion, 'ingress_ca_b64') %} -{%- if ingress_ca_mine %} -{%- set ca_pem = salt['hashutil.base64_b64decode'](ingress_ca_mine[pillar.metalk8s.ca.minion]) %} -{%- endif %} -{%- elif oidc_config.get('CAFile') %} +{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %} +{%- do oidc_config.update(pillar.kubernetes.apiServer.oidc) %} +{%- if oidc_config.get('CAFile') %} {%- set ca_pem = salt['file.read'](oidc_config.CAFile) %} {%- endif %} +{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %} +{%- do oidc_config.update({ + "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc", + "clientID": "oidc-auth-client", + "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt", + "usernameClaim": "email", + "groupsClaim": "groups", + }) %} +{%- set ingress_ca_mine = salt['mine.get'](pillar.metalk8s.ca.minion, 'ingress_ca_b64') %} +{%- if ingress_ca_mine %} +{%- set ca_pem = salt['hashutil.base64_b64decode'](ingress_ca_mine[pillar.metalk8s.ca.minion]) %} +{%- endif %} {%- endif %} @@ -84,7 +81,7 @@ Create kube-apiserver authentication configuration: url: {{ oidc_config.issuerURL }} audiences: - {{ oidc_config.clientID }} - certificateAuthority: {{ ca_pem }} + certificateAuthority: {{ ca_pem | tojson }} claimMappings: username: claim: {{ oidc_config.usernameClaim }}