security: disable anonymous auth on kube-apiserver (MK8S-187)

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Release 133.0.6 (in development)
 
+### Bug Fixes
+
+- Disable anonymous authentication on `kube-apiserver`, except for the kubelet
+  probe endpoints (`livez`, `readyz`, `healthz`)
+  (PR[#4900](https://github.com/scality/metalk8s/pull/4900))
+
 ## Release 133.0.5
 
 ### Enhancements

buildchain/buildchain/salt_tree.py

@@ -540,6 +540,7 @@ def _download_ui_operator_crds() -> str:
     Path("salt/metalk8s/kubernetes/apiserver/certs/server.sls"),
     Path("salt/metalk8s/kubernetes/apiserver/init.sls"),
     Path("salt/metalk8s/kubernetes/apiserver/installed.sls"),
+    Path("salt/metalk8s/kubernetes/apiserver/authnconfig.sls"),
     Path("salt/metalk8s/kubernetes/apiserver/cryptconfig.sls"),
     Path("salt/metalk8s/kubernetes/apiserver/kubeconfig.sls"),
     Path("salt/metalk8s/kubernetes/apiserver-proxy/files/apiserver-proxy.conf.j2"),

salt/metalk8s/defaults.yaml

@@ -151,6 +151,7 @@ kube_api:
   cert:
     server_signing_policy: kube_apiserver_server_policy
     client_signing_policy: kube_apiserver_client_policy
+  authn_config_path: /etc/kubernetes/authn_config.yaml
 
 etcd:
   ca:
@@ -224,6 +225,7 @@ certificates:
         watched: False
       apiserver-kubelet:
         path: /etc/kubernetes/pki/apiserver-kubelet-client.crt
+        key: /etc/kubernetes/pki/apiserver-kubelet-client.key
         renew:
           sls:
             - metalk8s.kubernetes.apiserver.certs.kubelet-client

salt/metalk8s/kubernetes/apiserver/authnconfig.sls

@@ -0,0 +1,99 @@
+# Generate the AuthenticationConfiguration file consumed by kube-apiserver
+# via --authentication-config.
+#
+# It carries:
+#   * `anonymous` -- limits anonymous access to the kubelet probe endpoints
+#     (/livez, /readyz, /healthz) so kubelet httpGet probes keep working,
+#     while every other path (/version, /api/*, ...) still requires
+#     authentication.
+#   * `jwt` -- the OIDC issuer (Dex by default, or a pillar override). This
+#     replaces the legacy --oidc-* command-line flags, which are mutually
+#     exclusive with --authentication-config in Kubernetes 1.32+
+#     (pkg/kubeapiserver/options/authentication.go: "authentication-config
+#     file and oidc-* flags are mutually exclusive").
+#
+# Relies on the AnonymousAuthConfigurableEndpoints and
+# StructuredAuthenticationConfiguration feature gates, both beta and
+# on-by-default in Kubernetes 1.32+.
+{%- from "metalk8s/map.jinja" import kube_api with context %}
+
+include:
+  - .installed
+  - metalk8s.addons.nginx-ingress.ca.advertised
+
+{%- set authn_config_path = kube_api.authn_config_path %}
+
+{#- Build the OIDC issuer config and resolve the matching CA PEM, mirroring the
+    historical --oidc-* selection logic that used to live in installed.sls.
+    AuthenticationConfiguration's `jwt[].issuer.certificateAuthority` field
+    expects PEM content inline (not a file path), so each branch resolves the
+    CA the way that fits its source:
+      * pillar override -- read the user-specified CAFile from the salt master.
+      * default Dex     -- pull the Ingress CA from the salt mine
+                           (`ingress_ca_b64`, published by
+                           `metalk8s.addons.nginx-ingress.ca.installed`), which
+                           avoids any ordering dependency on the on-disk file. #}
+{%- set oidc_config = {} %}
+{%- set ca_pem = '' %}
+{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %}
+{%-   do oidc_config.update(pillar.kubernetes.apiServer.oidc) %}
+{%-   if oidc_config.get('CAFile') %}
+{%-     set ca_pem = salt['file.read'](oidc_config.CAFile) %}
+{%-   endif %}
+{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %}
+{%-   do oidc_config.update({
+        "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc",
+        "clientID": "oidc-auth-client",
+        "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt",
+        "usernameClaim": "email",
+        "groupsClaim": "groups",
+      }) %}
+{%-   set ingress_ca_mine = salt['mine.get'](pillar.metalk8s.ca.minion, 'ingress_ca_b64') %}
+{%-   if ingress_ca_mine %}
+{%-     set ca_pem = salt['hashutil.base64_b64decode'](ingress_ca_mine[pillar.metalk8s.ca.minion]) %}
+{%-   endif %}
+{%- endif %}
+
+
+Create kube-apiserver authentication configuration:
+  file.serialize:
+    - name: {{ authn_config_path }}
+    - mode: '0600'
+    - user: root
+    - group: root
+    - makedirs: True
+    - dataset:
+        {#- TODO(MK8S-258): bump apiVersion to apiserver.config.k8s.io/v1 once metalk8s
+        pins Kubernetes >= 1.34. AuthenticationConfiguration is registered in
+        v1beta1 in 1.32/1.33 and promotes to v1 (GA) in 1.34.
+        it will also continue to be supported in v1beta1 until 1.36 #}
+        apiVersion: apiserver.config.k8s.io/v1beta1
+        kind: AuthenticationConfiguration
+        anonymous:
+          enabled: true
+          conditions:
+          - path: /livez
+          - path: /readyz
+          - path: /healthz
+        {%- if oidc_config and ca_pem %}
+        jwt:
+        - issuer:
+            url: {{ oidc_config.issuerURL }}
+            audiences:
+            - {{ oidc_config.clientID }}
+            certificateAuthority: {{ ca_pem | tojson }}
+          claimMappings:
+            username:
+              claim: {{ oidc_config.usernameClaim }}
+              prefix: "oidc:"
+            groups:
+              claim: {{ oidc_config.groupsClaim }}
+              prefix: "oidc:"
+          {%- if oidc_config.usernameClaim == 'email' %}
+          claimValidationRules:
+          - expression: "claims.?email_verified.orValue(true) == true"
+            message: "email_verified claim must be true when set"
+          {%- endif %}
+        {%- endif %}
+    - require_in:
+      - metalk8s: Create kube-apiserver Pod manifest

salt/metalk8s/kubernetes/apiserver/certs/kubelet-client.sls

@@ -1,7 +1,7 @@
 {%- from "metalk8s/map.jinja" import certificates with context %}
 {%- from "metalk8s/map.jinja" import kube_api with context %}
 
-{%- set private_key_path = "/etc/kubernetes/pki/apiserver-kubelet-client.key" %}
+{%- set private_key_path = certificates.client.files['apiserver-kubelet'].key %}
 
 include:
   - metalk8s.internal.m2crypto

salt/metalk8s/kubernetes/apiserver/init.sls

@@ -7,8 +7,10 @@
 # * installed     -> deploy apiserver manifest
 # * kubeconfig    -> create admin kubeconfig file
 # * cryptconfig   -> create apiserver encryption configuration
+# * authnconfig   -> create apiserver authentication configuration
 #
 include:
   - .installed
   - .kubeconfig
   - .cryptconfig
+  - .authnconfig

salt/metalk8s/kubernetes/apiserver/installed.sls

@@ -3,9 +3,10 @@
 {%- from "metalk8s/map.jinja" import coredns with context %}
 {%- from "metalk8s/map.jinja" import metalk8s with context %}
 {%- from "metalk8s/map.jinja" import networks with context %}
+{%- from "metalk8s/map.jinja" import kube_api with context %}
 
 {%- set encryption_k8s_path = "/etc/kubernetes/encryption.conf" %}
-
+{%- set authn_config_path = kube_api.authn_config_path %}
 include:
   - metalk8s.kubernetes.ca.advertised
   - metalk8s.kubernetes.sa.advertised
@@ -34,18 +35,9 @@ include:
 {%-   do feature_gates.append(feature ~ "=" ~ value) %}
 {%- endfor %}
 
-{%- set oidc_config = {} %}
-{%- if pillar.kubernetes.get("apiServer", {}).get("oidc") %}
-  {%- do oidc_config.update(pillar.kubernetes.apiServer.oidc) %}
-{%- elif pillar.addons.dex.enabled and salt.metalk8s_network.get_control_plane_ingress_endpoint() %}
-  {%- do oidc_config.update({
-    "issuerURL": salt.metalk8s_network.get_control_plane_ingress_endpoint() ~ "/oidc",
-    "clientID": "oidc-auth-client",
-    "CAFile": "/etc/metalk8s/pki/nginx-ingress/ca.crt",
-    "usernameClaim": "email",
-    "groupsClaim": "groups",
-  }) %}
-{%- endif %}
+{# OIDC is configured via the AuthenticationConfiguration file written by
+   .authnconfig (--oidc-* flags are mutually exclusive with
+   --authentication-config in Kubernetes 1.32+). #}
 
 {%- set pod_name = "kube-apiserver-" ~ grains.id %}
 {%- set last_pod_id = salt.cri.get_pod_id(
@@ -58,12 +50,13 @@ Create kube-apiserver Pod manifest:
     - source: salt://metalk8s/kubernetes/files/control-plane-manifest.yaml.j2
     - config_files:
         - {{ encryption_k8s_path }}
+        - {{ authn_config_path }}
         - {{ certificates.server.files.apiserver.path }}
         - /etc/kubernetes/pki/apiserver.key
         - {{ certificates.client.files['apiserver-etcd'].path }}
         - /etc/kubernetes/pki/apiserver-etcd-client.key
         - {{ certificates.client.files['apiserver-kubelet'].path }}
-        - /etc/kubernetes/pki/apiserver-kubelet-client.key
+        - {{ certificates.client.files['apiserver-kubelet'].key }}
         - /etc/kubernetes/pki/ca.crt
         - /etc/kubernetes/pki/etcd/ca.crt
         - /etc/kubernetes/pki/front-proxy-ca.crt
@@ -94,7 +87,7 @@ Create kube-apiserver Pod manifest:
           - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
           - --etcd-servers={{ etcd_servers | join(",") }}
           - --kubelet-client-certificate={{ certificates.client.files['apiserver-kubelet'].path }}
-          - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
+          - --kubelet-client-key={{ certificates.client.files['apiserver-kubelet'].key }}
           - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
           - --proxy-client-cert-file={{ certificates.client.files['front-proxy'].path }}
           - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
@@ -114,16 +107,8 @@ Create kube-apiserver Pod manifest:
           - --bind-address={{ host }}
           - --encryption-provider-config={{ encryption_k8s_path }}
           - --cors-allowed-origins=^.*$
-          {%- if oidc_config %}
-          - --oidc-issuer-url={{ oidc_config.issuerURL }}
-          - --oidc-client-id={{ oidc_config.clientID }}
-          - --oidc-ca-file={{ oidc_config.CAFile }}
-          - --oidc-username-claim={{ oidc_config.usernameClaim }}
-          - --oidc-groups-claim={{ oidc_config.groupsClaim }}
-          - '"--oidc-username-prefix=oidc:"'
-          - '"--oidc-groups-prefix=oidc:"'
-          {%- endif %}
           - --v={{ 2 if metalk8s.debug else 0 }}
+          - --authentication-config={{ authn_config_path }}
           {% if feature_gates %}
           - --feature-gates={{ feature_gates | join(",") }}
           {%- endif %}
@@ -132,6 +117,9 @@ Create kube-apiserver Pod manifest:
           - path: {{ encryption_k8s_path }}
             type: File
             name: k8s-encryption
+          - path: {{ authn_config_path }}
+            type: File
+            name: k8s-authn-config
           {%- if grains['os_family'] == 'RedHat' %}
           - path: /etc/pki/ca-trust
             name: etc-pki-ca-trust

tests/post/features/authentication.feature

@@ -21,6 +21,33 @@ Feature: Authentication is up and running
         When we perform a request on '/oidc/' on control-plane Ingress
         Then the server returns '404' with message '404 page not found'
 
+    Scenario: kube-apiserver rejects anonymous requests
+        Given the Kubernetes API is available
+        When we perform a request on '/api/kubernetes/version' on control-plane Ingress
+        Then the server returns '401' with message 'Unauthorized'
+
+    Scenario Outline: kube-apiserver allows anonymous access to <path>
+        Given the Kubernetes API is available
+        When we perform an anonymous request on the API server '<path>' endpoint
+        Then the server returns '200' with message 'ok'
+
+        Examples:
+            | path    |
+            | livez   |
+            | readyz  |
+            | healthz |
+
+    Scenario Outline: kube-apiserver accepts authenticated access to <path>
+        Given the Kubernetes API is available
+        When we perform an authenticated request on the API server '<path>' endpoint
+        Then the server returns '200' with message 'ok'
+
+        Examples:
+            | path    |
+            | livez   |
+            | readyz  |
+            | healthz |
+
     Scenario: Login to Dex using incorrect email
         Given the Kubernetes API is available
         And the control-plane Ingress path '/oidc' is available