security: disable anonymous auth on kube-apiserver (MK8S-187)

buildchain/buildchain/salt_tree.py

@@ -522,6 +522,7 @@ def _download_ui_operator_crds() -> str:
     Path("salt/metalk8s/kubernetes/apiserver/certs/server.sls"),
     Path("salt/metalk8s/kubernetes/apiserver/init.sls"),
     Path("salt/metalk8s/kubernetes/apiserver/installed.sls"),
+    Path("salt/metalk8s/kubernetes/apiserver/authnconfig.sls"),
     Path("salt/metalk8s/kubernetes/apiserver/cryptconfig.sls"),
     Path("salt/metalk8s/kubernetes/apiserver/kubeconfig.sls"),
     Path("salt/metalk8s/kubernetes/apiserver-proxy/files/apiserver-proxy.conf.j2"),

salt/metalk8s/defaults.yaml

@@ -150,6 +150,7 @@ kube_api:
   cert:
     server_signing_policy: kube_apiserver_server_policy
     client_signing_policy: kube_apiserver_client_policy
+  authn_config_path: /etc/kubernetes/authn_config.yaml
 
 etcd:
   ca:
@@ -223,6 +224,7 @@ certificates:
         watched: False
       apiserver-kubelet:
         path: /etc/kubernetes/pki/apiserver-kubelet-client.crt
+        key: /etc/kubernetes/pki/apiserver-kubelet-client.key
         renew:
           sls:
             - metalk8s.kubernetes.apiserver.certs.kubelet-client

salt/metalk8s/kubernetes/apiserver/authnconfig.sls

@@ -0,0 +1,102 @@
+# Generate the AuthenticationConfiguration file consumed by kube-apiserver
+# via --authentication-config.
+#
+# It carries:
+#   * `anonymous` -- limits anonymous access to the kubelet probe endpoints
+#     (/livez, /readyz, /healthz) so kubelet httpGet probes keep working,
+#     while every other path (/version, /api/*, ...) still requires
+#     authentication.
+#   * `jwt` -- the OIDC issuer (Dex by default, or a pillar override). This
+#     replaces the legacy --oidc-* command-line flags, which are mutually
+#     exclusive with --authentication-config in Kubernetes 1.32+
+#     (pkg/kubeapiserver/options/authentication.go: "authentication-config
+#     file and oidc-* flags are mutually exclusive").
+#
+# Relies on the AnonymousAuthConfigurableEndpoints and
+# StructuredAuthenticationConfiguration feature gates, both beta and
+# on-by-default in Kubernetes 1.32+.
+{%- from "metalk8s/map.jinja" import kube_api with context %}
+
+include:
+  - .installed
+  - metalk8s.addons.nginx-ingress.ca.advertised
+
+{%- set authn_config_path = kube_api.authn_config_path %}
+
+{#- Build the OIDC issuer config, mirroring the historical --oidc-* selection
+    logic that used to live in installed.sls. #}
+{%- set oidc_config = {} %}
+{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %}
+  {%- do oidc_config.update(pillar.kubernetes.apiServer.oidc) %}
+{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %}
+  {%- do oidc_config.update({
+    "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc",
+    "clientID": "oidc-auth-client",
+    "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt",
+    "usernameClaim": "email",
+    "groupsClaim": "groups",
+  }) %}
+{%- endif %}
+
+{#- AuthenticationConfiguration's `jwt[].issuer.certificateAuthority` field
+    expects PEM content inline, not a file path. For the default Dex case the
+    Ingress CA is published in the salt mine as `ingress_ca_b64` by
+    `metalk8s.addons.nginx-ingress.ca.installed`, which avoids any ordering
+    dependency on the on-disk file. For a pillar-provided OIDC override we
+    fall back to reading the user-specified CAFile from the salt master. #}
+{%- set ca_pem = '' %}
+{%- if oidc_config %}
+{%-   set ingress_ca_path = '/etc/metalk8s/pki/nginx-ingress/ca.crt' %}
+{%-   if oidc_config.get('CAFile') == ingress_ca_path %}
+{%-     set ingress_ca_mine = salt['mine.get'](pillar.metalk8s.ca.minion, 'ingress_ca_b64') %}
+{%-     if ingress_ca_mine %}
+{%-       set ca_pem = salt['hashutil.base64_b64decode'](ingress_ca_mine[pillar.metalk8s.ca.minion]) %}
+{%-     endif %}
+{%-   elif oidc_config.get('CAFile') %}
+{%-     set ca_pem = salt['file.read'](oidc_config.CAFile) %}
+{%-   endif %}
+{%- endif %}
+
+
+Create kube-apiserver authentication configuration:
+  file.serialize:
+    - name: {{ authn_config_path }}
+    - mode: '0600'
+    - user: root
+    - group: root
+    - makedirs: True
+    - dataset:
+        {#- TODO(MK8S-258): bump apiVersion to apiserver.config.k8s.io/v1 once metalk8s
+        pins Kubernetes >= 1.34. AuthenticationConfiguration is registered in
+        v1beta1 in 1.32/1.33 and promotes to v1 (GA) in 1.34.
+        it will also continue to be supported in v1beta1 until 1.36 #}
+        apiVersion: apiserver.config.k8s.io/v1beta1
+        kind: AuthenticationConfiguration
+        anonymous:
+          enabled: true
+          conditions:
+          - path: /livez
+          - path: /readyz
+          - path: /healthz
+        {%- if oidc_config and ca_pem %}
+        jwt:
+        - issuer:
+            url: {{ oidc_config.issuerURL }}
+            audiences:
+            - {{ oidc_config.clientID }}
+            certificateAuthority: {{ ca_pem }}
+          claimMappings:
+            username:
+              claim: {{ oidc_config.usernameClaim }}
+              prefix: "oidc:"
+            groups:
+              claim: {{ oidc_config.groupsClaim }}
+              prefix: "oidc:"
+          {%- if oidc_config.usernameClaim == 'email' %}
+          claimValidationRules:
+          - expression: "claims.?email_verified.orValue(true) == true"
+            message: "email_verified claim must be true when set"
+          {%- endif %}
+        {%- endif %}
+    - require_in:
+      - metalk8s: Create kube-apiserver Pod manifest

salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls

@@ -1,7 +1,7 @@
 {%- from "metalk8s/map.jinja" import certificates with context %}
 {%- from "metalk8s/map.jinja" import kube_api with context %}
 
-{%- set private_key_path = "/etc/kubernetes/pki/apiserver-kubelet-client.key" %}
+{%- set private_key_path = certificates.client.files['apiserver-kubelet'].key %}
 
 include:
   - metalk8s.internal.m2crypto

salt/metalk8s/kubernetes/apiserver/init.sls

@@ -7,8 +7,10 @@
 # * installed     -> deploy apiserver manifest
 # * kubeconfig    -> create admin kubeconfig file
 # * cryptconfig   -> create apiserver encryption configuration
+# * authnconfig   -> create apiserver authentication configuration
 #
 include:
   - .installed
   - .kubeconfig
   - .cryptconfig
+  - .authnconfig

salt/metalk8s/kubernetes/apiserver/installed.sls

@@ -3,9 +3,10 @@
 {%- from "metalk8s/map.jinja" import coredns with context %}
 {%- from "metalk8s/map.jinja" import metalk8s with context %}
 {%- from "metalk8s/map.jinja" import networks with context %}
+{%- from "metalk8s/map.jinja" import kube_api with context %}
 
 {%- set encryption_k8s_path = "/etc/kubernetes/encryption.conf" %}
-
+{%- set authn_config_path = kube_api.authn_config_path %}
 include:
   - metalk8s.kubernetes.ca.advertised
   - metalk8s.kubernetes.sa.advertised
@@ -34,18 +35,9 @@ include:
 {%-   do feature_gates.append(feature ~ "=" ~ value) %}
 {%- endfor %}
 
-{%- set oidc_config = {} %}
-{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %}
-  {%- do oidc_config.update(pillar.kubernetes.apiServer.oidc) %}
-{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %}
-  {%- do oidc_config.update({
-    "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc",
-    "clientID": "oidc-auth-client",
-    "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt",
-    "usernameClaim": "email",
-    "groupsClaim": "groups",
-  }) %}
-{%- endif %}
+{# OIDC is configured via the AuthenticationConfiguration file written by
+   .authnconfig (--oidc-* flags are mutually exclusive with
+   --authentication-config in Kubernetes 1.32+). #}
 
 {%- set pod_name = "kube-apiserver-" ~ grains.id %}
 {%- set last_pod_id = salt.cri.get_pod_id(
@@ -58,12 +50,13 @@ Create kube-apiserver Pod manifest:
     - source: salt://metalk8s/kubernetes/files/control-plane-manifest.yaml.j2
     - config_files:
         - {{ encryption_k8s_path }}
+        - {{ authn_config_path }}
         - {{ certificates.server.files.apiserver.path }}
         - /etc/kubernetes/pki/apiserver.key
         - {{ certificates.client.files['apiserver-etcd'].path }}
         - /etc/kubernetes/pki/apiserver-etcd-client.key
         - {{ certificates.client.files['apiserver-kubelet'].path }}
-        - /etc/kubernetes/pki/apiserver-kubelet-client.key
+        - {{ certificates.client.files['apiserver-kubelet'].key }}
         - /etc/kubernetes/pki/ca.crt
         - /etc/kubernetes/pki/etcd/ca.crt
         - /etc/kubernetes/pki/front-proxy-ca.crt
@@ -94,7 +87,7 @@ Create kube-apiserver Pod manifest:
           - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
           - --etcd-servers={{ etcd_servers | join(",") }}
           - --kubelet-client-certificate={{ certificates.client.files['apiserver-kubelet'].path }}
-          - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
+          - --kubelet-client-key={{ certificates.client.files['apiserver-kubelet'].key }}
           - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
           - --proxy-client-cert-file={{ certificates.client.files['front-proxy'].path }}
           - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
@@ -114,16 +107,8 @@ Create kube-apiserver Pod manifest:
           - --bind-address={{ host }}
           - --encryption-provider-config={{ encryption_k8s_path }}
           - --cors-allowed-origins=^.*$
-          {%- if oidc_config %}
-          - --oidc-issuer-url={{ oidc_config.issuerURL }}
-          - --oidc-client-id={{ oidc_config.clientID }}
-          - --oidc-ca-file={{ oidc_config.CAFile }}
-          - --oidc-username-claim={{ oidc_config.usernameClaim }}
-          - --oidc-groups-claim={{ oidc_config.groupsClaim }}
-          - '"--oidc-username-prefix=oidc:"'
-          - '"--oidc-groups-prefix=oidc:"'
-          {%- endif %}
           - --v={{ 2 if metalk8s.debug else 0 }}
+          - --authentication-config={{ authn_config_path }}
           {% if feature_gates %}
           - --feature-gates={{ feature_gates | join(",") }}
           {%- endif %}
@@ -132,6 +117,9 @@ Create kube-apiserver Pod manifest:
           - path: {{ encryption_k8s_path }}
             type: File
             name: k8s-encryption
+          - path: {{ authn_config_path }}
+            type: File
+            name: k8s-authn-config
           {%- if grains['os_family'] == 'RedHat' %}
           - path: /etc/pki/ca-trust
             name: etc-pki-ca-trust

tests/post/features/authentication.feature

@@ -21,6 +21,33 @@ Feature: Authentication is up and running
         When we perform a request on '/oidc/' on control-plane Ingress
         Then the server returns '404' with message '404 page not found'
 
+    Scenario: kube-apiserver rejects anonymous requests
+        Given the Kubernetes API is available
+        When we perform a request on '/api/kubernetes/version' on control-plane Ingress
+        Then the server returns '401' with message 'Unauthorized'
+
+    Scenario Outline: kube-apiserver allows anonymous access to <path>
+        Given the Kubernetes API is available
+        When we perform an anonymous request on the API server '<path>' endpoint
+        Then the server returns '200' with message 'ok'
+
+        Examples:
+            | path    |
+            | livez   |
+            | readyz  |
+            | healthz |
+
+    Scenario Outline: kube-apiserver accepts authenticated access to <path>
+        Given the Kubernetes API is available
+        When we perform an authenticated request on the API server '<path>' endpoint
+        Then the server returns '200' with message 'ok'
+
+        Examples:
+            | path    |
+            | livez   |
+            | readyz  |
+            | healthz |
+
     Scenario: Login to Dex using incorrect email
         Given the Kubernetes API is available
         And the control-plane Ingress path '/oidc' is available

tests/post/steps/test_authentication.py

@@ -36,6 +36,32 @@ def test_access_https_service(host):
     pass
 
 
+@scenario(
+    "../features/authentication.feature",
+    "kube-apiserver rejects anonymous requests",
+)
+def test_apiserver_rejects_anonymous(host):
+    pass
+
+
+@scenario(
+    "../features/authentication.feature",
+    "kube-apiserver allows anonymous access to <path>",
+    example_converters={"path": str},
+)
+def test_apiserver_allows_anonymous_health(host):
+    pass
+
+
+@scenario(
+    "../features/authentication.feature",
+    "kube-apiserver accepts authenticated access to <path>",
+    example_converters={"path": str},
+)
+def test_apiserver_accepts_authenticated_health(host):
+    pass
+
+
 @scenario("../features/authentication.feature", "Login to Dex using incorrect email")
 def test_failed_login(host):
     pass
@@ -109,6 +135,46 @@ def perform_request(host, context, control_plane_ingress_ep, path):
         pytest.fail("Failed to access oidc url path with error: {}".format(exc))
 
 
+@when("we perform an anonymous request on the API server '<path>' endpoint")
+def perform_anonymous_apiserver_request(context, control_plane_ip, path):
+    """Hit kube-apiserver directly on :6443 with no credentials.
+
+    Bypasses the control-plane Ingress so the path the apiserver sees is
+    exactly `/<path>`, with no rewrite ambiguity. `path` is the example
+    value (e.g. "livez") supplied by the Scenario Outline. We use the
+    literal-text decorator style here -- in pytest-bdd 3.2.1 only this
+    style substitutes `<placeholder>` references; parsers.parse(...) keeps
+    them literal.
+    """
+    session = utils.requests_retry_session()
+    try:
+        context["response"] = session.get(
+            "https://{ip}:6443/{path}".format(ip=control_plane_ip, path=path),
+            verify=False,
+        )
+    except requests.exceptions.ConnectionError as exc:
+        pytest.fail("Failed to access API server with error: {}".format(exc))
+
+
+@when("we perform an authenticated request on the API server '<path>' endpoint")
+def perform_authenticated_request(context, control_plane_ip, k8s_client, path):
+    """Hit kube-apiserver directly on :6443 with the admin client cert.
+
+    Same notes as the anonymous variant on URL construction and on the
+    decorator style.
+    """
+    config = k8s_client.client.configuration
+    session = utils.requests_retry_session()
+    try:
+        context["response"] = session.get(
+            "https://{ip}:6443/{path}".format(ip=control_plane_ip, path=path),
+            verify=config.ssl_ca_cert,
+            cert=(config.cert_file, config.key_file),
+        )
+    except requests.exceptions.ConnectionError as exc:
+        pytest.fail("Failed to access API server with error: {}".format(exc))
+
+
 # }}}
 # Then {{{
 
@@ -151,8 +217,46 @@ def _get_openID_config():
 def server_returns(host, context, status_code, status_message):
     response = context.get("response")
     assert response is not None
-    assert response.status_code == int(status_code)
-    assert response.text.rstrip("\n") == status_message
+
+    expected_code = int(status_code)
+    actual_url = response.request.url if response.request else "<unknown>"
+    actual_body_excerpt = response.text[:500].replace("\n", "\\n")
+
+    # kube-apiserver returns either:
+    #   * a plain-text body (e.g. /livez returns "ok\n"), or
+    #   * a structured `Status` JSON object (e.g. on a 401 with
+    #     AuthenticationConfiguration in use, the body is
+    #     {"kind":"Status",...,"message":"Unauthorized","code":401}).
+    # Compare against `message` for the JSON case, against the raw text
+    # otherwise.
+    try:
+        parsed = response.json()
+    except ValueError:
+        parsed = None
+    actual_message = (
+        parsed.get("message")
+        if isinstance(parsed, dict) and parsed.get("kind") == "Status"
+        else response.text.rstrip("\n")
+    )
+
+    assert response.status_code == expected_code, (
+        "Expected HTTP {expected} but got {actual} from {url}. "
+        "Body excerpt: {body}".format(
+            expected=expected_code,
+            actual=response.status_code,
+            url=actual_url,
+            body=actual_body_excerpt,
+        )
+    )
+    assert actual_message == status_message, (
+        "Expected message {expected!r} but got {actual!r} from {url}. "
+        "Body excerpt: {body}".format(
+            expected=status_message,
+            actual=actual_message,
+            url=actual_url,
+            body=actual_body_excerpt,
+        )
+    )
 
 
 #  }}}