diff --git a/CHANGELOG.md b/CHANGELOG.md index 0534e5962e..cf96ccde31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Release 133.0.6 (in development) +### Bug Fixes + +- Disable anonymous authentication on `kube-apiserver`, except for the kubelet + probe endpoints (`livez`, `readyz`, `healthz`) + (PR[#4900](https://github.com/scality/metalk8s/pull/4900)) + ## Release 133.0.5 ### Enhancements diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index ee597238ce..883c092d53 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -540,6 +540,7 @@ def _download_ui_operator_crds() -> str: Path("salt/metalk8s/kubernetes/apiserver/certs/server.sls"), Path("salt/metalk8s/kubernetes/apiserver/init.sls"), Path("salt/metalk8s/kubernetes/apiserver/installed.sls"), + Path("salt/metalk8s/kubernetes/apiserver/authnconfig.sls"), Path("salt/metalk8s/kubernetes/apiserver/cryptconfig.sls"), Path("salt/metalk8s/kubernetes/apiserver/kubeconfig.sls"), Path("salt/metalk8s/kubernetes/apiserver-proxy/files/apiserver-proxy.conf.j2"), diff --git a/salt/metalk8s/defaults.yaml b/salt/metalk8s/defaults.yaml index 10b0fd4d14..b55a2e31aa 100644 --- a/salt/metalk8s/defaults.yaml +++ b/salt/metalk8s/defaults.yaml @@ -151,6 +151,7 @@ kube_api: cert: server_signing_policy: kube_apiserver_server_policy client_signing_policy: kube_apiserver_client_policy + authn_config_path: /etc/kubernetes/authn_config.yaml etcd: ca: @@ -224,6 +225,7 @@ certificates: watched: False apiserver-kubelet: path: /etc/kubernetes/pki/apiserver-kubelet-client.crt + key: /etc/kubernetes/pki/apiserver-kubelet-client.key renew: sls: - metalk8s.kubernetes.apiserver.certs.kubelet-client diff --git a/salt/metalk8s/kubernetes/apiserver/authnconfig.sls b/salt/metalk8s/kubernetes/apiserver/authnconfig.sls new file mode 100644 index 0000000000..92235b42ad --- /dev/null +++ b/salt/metalk8s/kubernetes/apiserver/authnconfig.sls @@ -0,0 +1,99 @@ +# Generate the AuthenticationConfiguration file consumed by kube-apiserver +# via --authentication-config. +# +# It carries: +# * `anonymous` -- limits anonymous access to the kubelet probe endpoints +# (/livez, /readyz, /healthz) so kubelet httpGet probes keep working, +# while every other path (/version, /api/*, ...) still requires +# authentication. +# * `jwt` -- the OIDC issuer (Dex by default, or a pillar override). This +# replaces the legacy --oidc-* command-line flags, which are mutually +# exclusive with --authentication-config in Kubernetes 1.32+ +# (pkg/kubeapiserver/options/authentication.go: "authentication-config +# file and oidc-* flags are mutually exclusive"). +# +# Relies on the AnonymousAuthConfigurableEndpoints and +# StructuredAuthenticationConfiguration feature gates, both beta and +# on-by-default in Kubernetes 1.32+. +{%- from "metalk8s/map.jinja" import kube_api with context %} + +include: + - .installed + - metalk8s.addons.nginx-ingress.ca.advertised + +{%- set authn_config_path = kube_api.authn_config_path %} + +{#- Build the OIDC issuer config and resolve the matching CA PEM, mirroring the + historical --oidc-* selection logic that used to live in installed.sls. + AuthenticationConfiguration's `jwt[].issuer.certificateAuthority` field + expects PEM content inline (not a file path), so each branch resolves the + CA the way that fits its source: + * pillar override -- read the user-specified CAFile from the salt master. + * default Dex -- pull the Ingress CA from the salt mine + (`ingress_ca_b64`, published by + `metalk8s.addons.nginx-ingress.ca.installed`), which + avoids any ordering dependency on the on-disk file. #} +{%- set oidc_config = {} %} +{%- set ca_pem = '' %} +{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %} +{%- do oidc_config.update(pillar.kubernetes.apiServer.oidc) %} +{%- if oidc_config.get('CAFile') %} +{%- set ca_pem = salt['file.read'](oidc_config.CAFile) %} +{%- endif %} +{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %} +{%- do oidc_config.update({ + "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc", + "clientID": "oidc-auth-client", + "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt", + "usernameClaim": "email", + "groupsClaim": "groups", + }) %} +{%- set ingress_ca_mine = salt['mine.get'](pillar.metalk8s.ca.minion, 'ingress_ca_b64') %} +{%- if ingress_ca_mine %} +{%- set ca_pem = salt['hashutil.base64_b64decode'](ingress_ca_mine[pillar.metalk8s.ca.minion]) %} +{%- endif %} +{%- endif %} + + +Create kube-apiserver authentication configuration: + file.serialize: + - name: {{ authn_config_path }} + - mode: '0600' + - user: root + - group: root + - makedirs: True + - dataset: + {#- TODO(MK8S-258): bump apiVersion to apiserver.config.k8s.io/v1 once metalk8s + pins Kubernetes >= 1.34. AuthenticationConfiguration is registered in + v1beta1 in 1.32/1.33 and promotes to v1 (GA) in 1.34. + it will also continue to be supported in v1beta1 until 1.36 #} + apiVersion: apiserver.config.k8s.io/v1beta1 + kind: AuthenticationConfiguration + anonymous: + enabled: true + conditions: + - path: /livez + - path: /readyz + - path: /healthz + {%- if oidc_config and ca_pem %} + jwt: + - issuer: + url: {{ oidc_config.issuerURL }} + audiences: + - {{ oidc_config.clientID }} + certificateAuthority: {{ ca_pem | tojson }} + claimMappings: + username: + claim: {{ oidc_config.usernameClaim }} + prefix: "oidc:" + groups: + claim: {{ oidc_config.groupsClaim }} + prefix: "oidc:" + {%- if oidc_config.usernameClaim == 'email' %} + claimValidationRules: + - expression: "claims.?email_verified.orValue(true) == true" + message: "email_verified claim must be true when set" + {%- endif %} + {%- endif %} + - require_in: + - metalk8s: Create kube-apiserver Pod manifest diff --git a/salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls b/salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls index dd46a2763e..269fcda883 100644 --- a/salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls +++ b/salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls @@ -1,7 +1,7 @@ {%- from "metalk8s/map.jinja" import certificates with context %} {%- from "metalk8s/map.jinja" import kube_api with context %} -{%- set private_key_path = "/etc/kubernetes/pki/apiserver-kubelet-client.key" %} +{%- set private_key_path = certificates.client.files['apiserver-kubelet'].key %} include: - metalk8s.internal.m2crypto diff --git a/salt/metalk8s/kubernetes/apiserver/init.sls b/salt/metalk8s/kubernetes/apiserver/init.sls index d47b3f8850..68040b055e 100644 --- a/salt/metalk8s/kubernetes/apiserver/init.sls +++ b/salt/metalk8s/kubernetes/apiserver/init.sls @@ -7,8 +7,10 @@ # * installed -> deploy apiserver manifest # * kubeconfig -> create admin kubeconfig file # * cryptconfig -> create apiserver encryption configuration +# * authnconfig -> create apiserver authentication configuration # include: - .installed - .kubeconfig - .cryptconfig + - .authnconfig diff --git a/salt/metalk8s/kubernetes/apiserver/installed.sls b/salt/metalk8s/kubernetes/apiserver/installed.sls index a9a0afc368..5b8284184b 100644 --- a/salt/metalk8s/kubernetes/apiserver/installed.sls +++ b/salt/metalk8s/kubernetes/apiserver/installed.sls @@ -3,9 +3,10 @@ {%- from "metalk8s/map.jinja" import coredns with context %} {%- from "metalk8s/map.jinja" import metalk8s with context %} {%- from "metalk8s/map.jinja" import networks with context %} +{%- from "metalk8s/map.jinja" import kube_api with context %} {%- set encryption_k8s_path = "/etc/kubernetes/encryption.conf" %} - +{%- set authn_config_path = kube_api.authn_config_path %} include: - metalk8s.kubernetes.ca.advertised - metalk8s.kubernetes.sa.advertised @@ -34,18 +35,9 @@ include: {%- do feature_gates.append(feature ~ "=" ~ value) %} {%- endfor %} -{%- set oidc_config = {} %} -{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %} - {%- do oidc_config.update(pillar.kubernetes.apiServer.oidc) %} -{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %} - {%- do oidc_config.update({ - "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc", - "clientID": "oidc-auth-client", - "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt", - "usernameClaim": "email", - "groupsClaim": "groups", - }) %} -{%- endif %} +{# OIDC is configured via the AuthenticationConfiguration file written by + .authnconfig (--oidc-* flags are mutually exclusive with + --authentication-config in Kubernetes 1.32+). #} {%- set pod_name = "kube-apiserver-" ~ grains.id %} {%- set last_pod_id = salt.cri.get_pod_id( @@ -58,12 +50,13 @@ Create kube-apiserver Pod manifest: - source: salt://metalk8s/kubernetes/files/control-plane-manifest.yaml.j2 - config_files: - {{ encryption_k8s_path }} + - {{ authn_config_path }} - {{ certificates.server.files.apiserver.path }} - /etc/kubernetes/pki/apiserver.key - {{ certificates.client.files['apiserver-etcd'].path }} - /etc/kubernetes/pki/apiserver-etcd-client.key - {{ certificates.client.files['apiserver-kubelet'].path }} - - /etc/kubernetes/pki/apiserver-kubelet-client.key + - {{ certificates.client.files['apiserver-kubelet'].key }} - /etc/kubernetes/pki/ca.crt - /etc/kubernetes/pki/etcd/ca.crt - /etc/kubernetes/pki/front-proxy-ca.crt @@ -94,7 +87,7 @@ Create kube-apiserver Pod manifest: - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key - --etcd-servers={{ etcd_servers | join(",") }} - --kubelet-client-certificate={{ certificates.client.files['apiserver-kubelet'].path }} - - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key + - --kubelet-client-key={{ certificates.client.files['apiserver-kubelet'].key }} - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --proxy-client-cert-file={{ certificates.client.files['front-proxy'].path }} - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key @@ -114,16 +107,8 @@ Create kube-apiserver Pod manifest: - --bind-address={{ host }} - --encryption-provider-config={{ encryption_k8s_path }} - --cors-allowed-origins=^.*$ - {%- if oidc_config %} - - --oidc-issuer-url={{ oidc_config.issuerURL }} - - --oidc-client-id={{ oidc_config.clientID }} - - --oidc-ca-file={{ oidc_config.CAFile }} - - --oidc-username-claim={{ oidc_config.usernameClaim }} - - --oidc-groups-claim={{ oidc_config.groupsClaim }} - - '"--oidc-username-prefix=oidc:"' - - '"--oidc-groups-prefix=oidc:"' - {%- endif %} - --v={{ 2 if metalk8s.debug else 0 }} + - --authentication-config={{ authn_config_path }} {% if feature_gates %} - --feature-gates={{ feature_gates | join(",") }} {%- endif %} @@ -132,6 +117,9 @@ Create kube-apiserver Pod manifest: - path: {{ encryption_k8s_path }} type: File name: k8s-encryption + - path: {{ authn_config_path }} + type: File + name: k8s-authn-config {%- if grains['os_family'] == 'RedHat' %} - path: /etc/pki/ca-trust name: etc-pki-ca-trust diff --git a/tests/post/features/authentication.feature b/tests/post/features/authentication.feature index 699ca3b4b1..ee82a22ab8 100644 --- a/tests/post/features/authentication.feature +++ b/tests/post/features/authentication.feature @@ -21,6 +21,33 @@ Feature: Authentication is up and running When we perform a request on '/oidc/' on control-plane Ingress Then the server returns '404' with message '404 page not found' + Scenario: kube-apiserver rejects anonymous requests + Given the Kubernetes API is available + When we perform a request on '/api/kubernetes/version' on control-plane Ingress + Then the server returns '401' with message 'Unauthorized' + + Scenario Outline: kube-apiserver allows anonymous access to + Given the Kubernetes API is available + When we perform an anonymous request on the API server '' endpoint + Then the server returns '200' with message 'ok' + + Examples: + | path | + | livez | + | readyz | + | healthz | + + Scenario Outline: kube-apiserver accepts authenticated access to + Given the Kubernetes API is available + When we perform an authenticated request on the API server '' endpoint + Then the server returns '200' with message 'ok' + + Examples: + | path | + | livez | + | readyz | + | healthz | + Scenario: Login to Dex using incorrect email Given the Kubernetes API is available And the control-plane Ingress path '/oidc' is available diff --git a/tests/post/steps/test_authentication.py b/tests/post/steps/test_authentication.py index 97c4ca556e..b43b1e2d30 100644 --- a/tests/post/steps/test_authentication.py +++ b/tests/post/steps/test_authentication.py @@ -36,6 +36,32 @@ def test_access_https_service(host): pass +@scenario( + "../features/authentication.feature", + "kube-apiserver rejects anonymous requests", +) +def test_apiserver_rejects_anonymous(host): + pass + + +@scenario( + "../features/authentication.feature", + "kube-apiserver allows anonymous access to ", + example_converters={"path": str}, +) +def test_apiserver_allows_anonymous_health(host): + pass + + +@scenario( + "../features/authentication.feature", + "kube-apiserver accepts authenticated access to ", + example_converters={"path": str}, +) +def test_apiserver_accepts_authenticated_health(host): + pass + + @scenario("../features/authentication.feature", "Login to Dex using incorrect email") def test_failed_login(host): pass @@ -109,6 +135,46 @@ def perform_request(host, context, control_plane_ingress_ep, path): pytest.fail("Failed to access oidc url path with error: {}".format(exc)) +@when("we perform an anonymous request on the API server '' endpoint") +def perform_anonymous_apiserver_request(context, control_plane_ip, path): + """Hit kube-apiserver directly on :6443 with no credentials. + + Bypasses the control-plane Ingress so the path the apiserver sees is + exactly `/`, with no rewrite ambiguity. `path` is the example + value (e.g. "livez") supplied by the Scenario Outline. We use the + literal-text decorator style here -- in pytest-bdd 3.2.1 only this + style substitutes `` references; parsers.parse(...) keeps + them literal. + """ + session = utils.requests_retry_session() + try: + context["response"] = session.get( + "https://{ip}:6443/{path}".format(ip=control_plane_ip, path=path), + verify=False, + ) + except requests.exceptions.ConnectionError as exc: + pytest.fail("Failed to access API server with error: {}".format(exc)) + + +@when("we perform an authenticated request on the API server '' endpoint") +def perform_authenticated_request(context, control_plane_ip, k8s_client, path): + """Hit kube-apiserver directly on :6443 with the admin client cert. + + Same notes as the anonymous variant on URL construction and on the + decorator style. + """ + config = k8s_client.client.configuration + session = utils.requests_retry_session() + try: + context["response"] = session.get( + "https://{ip}:6443/{path}".format(ip=control_plane_ip, path=path), + verify=config.ssl_ca_cert, + cert=(config.cert_file, config.key_file), + ) + except requests.exceptions.ConnectionError as exc: + pytest.fail("Failed to access API server with error: {}".format(exc)) + + # }}} # Then {{{ @@ -151,8 +217,46 @@ def _get_openID_config(): def server_returns(host, context, status_code, status_message): response = context.get("response") assert response is not None - assert response.status_code == int(status_code) - assert response.text.rstrip("\n") == status_message + + expected_code = int(status_code) + actual_url = response.request.url if response.request else "" + actual_body_excerpt = response.text[:500].replace("\n", "\\n") + + # kube-apiserver returns either: + # * a plain-text body (e.g. /livez returns "ok\n"), or + # * a structured `Status` JSON object (e.g. on a 401 with + # AuthenticationConfiguration in use, the body is + # {"kind":"Status",...,"message":"Unauthorized","code":401}). + # Compare against `message` for the JSON case, against the raw text + # otherwise. + try: + parsed = response.json() + except ValueError: + parsed = None + actual_message = ( + parsed.get("message") + if isinstance(parsed, dict) and parsed.get("kind") == "Status" + else response.text.rstrip("\n") + ) + + assert response.status_code == expected_code, ( + "Expected HTTP {expected} but got {actual} from {url}. " + "Body excerpt: {body}".format( + expected=expected_code, + actual=response.status_code, + url=actual_url, + body=actual_body_excerpt, + ) + ) + assert actual_message == status_message, ( + "Expected message {expected!r} but got {actual!r} from {url}. " + "Body excerpt: {body}".format( + expected=status_message, + actual=actual_message, + url=actual_url, + body=actual_body_excerpt, + ) + ) # }}}