From 473bac4941bf4f84a680b797f57cda4b7245cbf1 Mon Sep 17 00:00:00 2001 From: Matt Boersma Date: Thu, 14 May 2026 14:52:21 -0600 Subject: [PATCH 1/2] test/e2e: add optional Azure Linux 4 cluster e2e test --- .../test/ci/cluster-template-prow-azl4.yaml | 483 +++++++++ ...cluster-template-prow-ci-version-azl4.yaml | 962 ++++++++++++++++++ .../test/ci/prow-azl4/kustomization.yaml | 36 + .../azuremachinetemplate-azl4-image.yaml | 25 + .../cloud-provider-azure-cacertdir.yaml | 12 + .../cloud-provider-azure-ci-cacertdir.yaml | 23 + .../prow-azl4/patches/controller-manager.yaml | 46 + .../disable-vm-bootstrap-extension.yaml | 17 + .../patches/kubeadm-config-template-azl4.yaml | 63 ++ .../patches/remove-marketplace-image.yaml | 19 + .../prow-ci-version-azl4/kustomization.yaml | 26 + test/e2e/azure_test.go | 89 ++ test/e2e/config/azure-dev.yaml | 4 + 13 files changed, 1805 insertions(+) create mode 100644 templates/test/ci/cluster-template-prow-azl4.yaml create mode 100644 templates/test/ci/cluster-template-prow-ci-version-azl4.yaml create mode 100644 templates/test/ci/prow-azl4/kustomization.yaml create mode 100644 templates/test/ci/prow-azl4/patches/azuremachinetemplate-azl4-image.yaml create mode 100644 templates/test/ci/prow-azl4/patches/cloud-provider-azure-cacertdir.yaml create mode 100644 templates/test/ci/prow-azl4/patches/cloud-provider-azure-ci-cacertdir.yaml create mode 100644 templates/test/ci/prow-azl4/patches/controller-manager.yaml create mode 100644 templates/test/ci/prow-azl4/patches/disable-vm-bootstrap-extension.yaml create mode 100644 templates/test/ci/prow-azl4/patches/kubeadm-config-template-azl4.yaml create mode 100644 templates/test/ci/prow-azl4/patches/remove-marketplace-image.yaml create mode 100644 templates/test/ci/prow-ci-version-azl4/kustomization.yaml diff --git a/templates/test/ci/cluster-template-prow-azl4.yaml b/templates/test/ci/cluster-template-prow-azl4.yaml new file mode 100644 index 00000000000..161d94655f1 --- /dev/null +++ b/templates/test/ci/cluster-template-prow-azl4.yaml @@ -0,0 +1,483 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + service-account-issuer: ${SERVICE_ACCOUNT_ISSUER:-https://kubernetes.default.svc.cluster.local} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Allow Azure service IP addresses (required for Azure resources) + iptables -A INPUT -s 168.63.129.16 -j ACCEPT + iptables -A OUTPUT -d 168.63.129.16 -j ACCEPT + + # Kubernetes API Server (port 6443) - bound to all IPv6 interfaces, needs external access + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + + # etcd server communication + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + + # Allow traffic to Kubernetes service network (10.96.0.0/12) + iptables -A OUTPUT -d 10.96.0.0/12 -j ACCEPT + iptables -A INPUT -s 10.96.0.0/12 -j ACCEPT + + # Allow traffic to/from node network (10.1.0.0/24) + iptables -A OUTPUT -d 10.1.0.0/24 -j ACCEPT + iptables -A INPUT -s 10.1.0.0/24 -j ACCEPT + + # Allow traffic to/from Calico pod network + iptables -A OUTPUT -d 192.168.0.0/24 -j ACCEPT + iptables -A INPUT -s 192.168.0.0/24 -j ACCEPT + + # Save the rules following Azure Linux 4 approach + iptables-save > /etc/systemd/scripts/ip4save + owner: root:root + path: /tmp/azl4-setup.sh + permissions: "0744" + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - bash -c /tmp/azl4-setup.sh + verbosity: 10 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-4-gen2 + version: ${AZL4_VERSION} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: azure:///subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: {} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + identity: UserAssigned + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-4-gen2 + version: ${AZL4_VERSION} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: azure:///subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Allow Azure service IP addresses (required for Azure resources) + iptables -A INPUT -s 168.63.129.16 -j ACCEPT + iptables -A OUTPUT -d 168.63.129.16 -j ACCEPT + + # Allow localhost traffic + iptables -A INPUT -i lo -j ACCEPT + iptables -A OUTPUT -o lo -j ACCEPT + + # Allow established and related connections + iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT + iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT + + # SSH (port 22) + # iptables -A INPUT -p tcp --dport 22 -j ACCEPT + + # Kubelet API (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + + # Allow traffic to Kubernetes service network (10.96.0.0/12) + iptables -A OUTPUT -d 10.96.0.0/12 -j ACCEPT + iptables -A INPUT -s 10.96.0.0/12 -j ACCEPT + + # Allow traffic to/from Calico pod network (192.168.0.0/16) + iptables -A OUTPUT -d 192.168.0.0/16 -j ACCEPT + iptables -A INPUT -s 192.168.0.0/16 -j ACCEPT + + # Allow traffic to/from node network (10.1.0.0/24) + iptables -A OUTPUT -d 10.1.0.0/24 -j ACCEPT + iptables -A INPUT -s 10.1.0.0/24 -j ACCEPT + + # Calico networking requirements + # Calico Typha (port 5473) + iptables -A INPUT -p tcp --dport 5473 -j ACCEPT + + # VXLAN for overlay networking (port 4789 UDP) + iptables -A INPUT -p udp --dport 4789 -j ACCEPT + + # BGP for node-to-node communication (port 179) + iptables -A INPUT -p tcp --d port 179 -j ACCEPT + + # DNS (port 53) + iptables -A INPUT -p udp --dport 53 -j ACCEPT + iptables -A OUTPUT -p udp --dport 53 -j ACCEPT + + # Save the rules following Azure Linux 4 approach + iptables-save > /etc/systemd/scripts/ip4save + owner: root:root + path: /tmp/azl4-setup.sh + permissions: "0744" + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - bash -c /tmp/azl4-setup.sh +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + windowsDataplane: HNS + mtu: 1350 + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: capzcicommunity.azurecr.io + serviceCIDRs: + - 10.96.0.0/12 # must match cluster service CIDR (this is the default) + # Image and registry configuration for the tigera/operator pod + tigeraOperator: + image: tigera/operator + registry: capzcicommunity.azurecr.io + calicoctl: + image: capzcicommunity.azurecr.io/calico/ctl + # when kubernetesServiceEndpoint (required for windows) is added + # DNS configuration is needed to look up the api server name properly + # https://github.com/projectcalico/calico/issues/9536 + dnsConfig: + nameservers: + - 127.0.0.53 + options: + - name: edns0 + - name: trust-ad + kubernetesServiceEndpoint: + host: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + port: "{{ .Cluster.spec.controlPlaneEndpoint.port }}" + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" diff --git a/templates/test/ci/cluster-template-prow-ci-version-azl4.yaml b/templates/test/ci/cluster-template-prow-ci-version-azl4.yaml new file mode 100644 index 00000000000..57486452fbd --- /dev/null +++ b/templates/test/ci/cluster-template-prow-ci-version-azl4.yaml @@ -0,0 +1,962 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + metrics-server: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + service-account-issuer: ${SERVICE_ACCOUNT_ISSUER:-https://kubernetes.default.svc.cluster.local} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + kubernetesVersion: ci/${CI_VERSION} + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Allow Azure service IP addresses (required for Azure resources) + iptables -A INPUT -s 168.63.129.16 -j ACCEPT + iptables -A OUTPUT -d 168.63.129.16 -j ACCEPT + + # Kubernetes API Server (port 6443) - bound to all IPv6 interfaces, needs external access + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + + # etcd server communication + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + + # Allow traffic to Kubernetes service network (10.96.0.0/12) + iptables -A OUTPUT -d 10.96.0.0/12 -j ACCEPT + iptables -A INPUT -s 10.96.0.0/12 -j ACCEPT + + # Allow traffic to/from node network (10.1.0.0/24) + iptables -A OUTPUT -d 10.1.0.0/24 -j ACCEPT + iptables -A INPUT -s 10.1.0.0/24 -j ACCEPT + + # Allow traffic to/from Calico pod network + iptables -A OUTPUT -d 192.168.0.0/24 -j ACCEPT + iptables -A INPUT -s 192.168.0.0/24 -j ACCEPT + + # Save the rules following Azure Linux 4 approach + iptables-save > /etc/systemd/scripts/ip4save + owner: root:root + path: /tmp/azl4-setup.sh + permissions: "0744" + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + CI_VERSION=${CI_VERSION} + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${CI_VERSION}" ]]; then + echo "Detected Kubernetes $${CI_VERSION} via kubectl version, nothing to do" + exit 0 + fi + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing version $${CI_VERSION}" + CI_URL="https://dl.k8s.io/ci/$${CI_VERSION}/bin/linux/amd64" + # Set CI_URL to the released binaries for actually released versions. + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${CI_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget --inet4-only "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + + systemctl restart kubelet + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${CI_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget --inet4-only "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${CI_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${CI_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${CI_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - bash -c /tmp/azl4-setup.sh + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-4-gen2 + version: ${AZL4_VERSION} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: azure:///subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: {} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + identity: UserAssigned + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-4-gen2 + version: ${AZL4_VERSION} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: azure:///subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Allow Azure service IP addresses (required for Azure resources) + iptables -A INPUT -s 168.63.129.16 -j ACCEPT + iptables -A OUTPUT -d 168.63.129.16 -j ACCEPT + + # Allow localhost traffic + iptables -A INPUT -i lo -j ACCEPT + iptables -A OUTPUT -o lo -j ACCEPT + + # Allow established and related connections + iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT + iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT + + # SSH (port 22) + # iptables -A INPUT -p tcp --dport 22 -j ACCEPT + + # Kubelet API (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + + # Allow traffic to Kubernetes service network (10.96.0.0/12) + iptables -A OUTPUT -d 10.96.0.0/12 -j ACCEPT + iptables -A INPUT -s 10.96.0.0/12 -j ACCEPT + + # Allow traffic to/from Calico pod network (192.168.0.0/16) + iptables -A OUTPUT -d 192.168.0.0/16 -j ACCEPT + iptables -A INPUT -s 192.168.0.0/16 -j ACCEPT + + # Allow traffic to/from node network (10.1.0.0/24) + iptables -A OUTPUT -d 10.1.0.0/24 -j ACCEPT + iptables -A INPUT -s 10.1.0.0/24 -j ACCEPT + + # Calico networking requirements + # Calico Typha (port 5473) + iptables -A INPUT -p tcp --dport 5473 -j ACCEPT + + # VXLAN for overlay networking (port 4789 UDP) + iptables -A INPUT -p udp --dport 4789 -j ACCEPT + + # BGP for node-to-node communication (port 179) + iptables -A INPUT -p tcp --d port 179 -j ACCEPT + + # DNS (port 53) + iptables -A INPUT -p udp --dport 53 -j ACCEPT + iptables -A OUTPUT -p udp --dport 53 -j ACCEPT + + # Save the rules following Azure Linux 4 approach + iptables-save > /etc/systemd/scripts/ip4save + owner: root:root + path: /tmp/azl4-setup.sh + permissions: "0744" + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + CI_VERSION=${CI_VERSION} + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${CI_VERSION}" ]]; then + echo "Detected Kubernetes $${CI_VERSION} via kubectl version, nothing to do" + exit 0 + fi + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing version $${CI_VERSION}" + CI_URL="https://dl.k8s.io/ci/$${CI_VERSION}/bin/linux/amd64" + # Set CI_URL to the released binaries for actually released versions. + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${CI_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget --inet4-only "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + + systemctl restart kubelet + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${CI_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget --inet4-only "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${CI_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${CI_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${CI_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - bash -c /tmp/azl4-setup.sh + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + windowsDataplane: HNS + mtu: 1350 + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: capzcicommunity.azurecr.io + serviceCIDRs: + - 10.96.0.0/12 # must match cluster service CIDR (this is the default) + # Image and registry configuration for the tigera/operator pod + tigeraOperator: + image: tigera/operator + registry: capzcicommunity.azurecr.io + calicoctl: + image: capzcicommunity.azurecr.io/calico/ctl + # when kubernetesServiceEndpoint (required for windows) is added + # DNS configuration is needed to look up the api server name properly + # https://github.com/projectcalico/calico/issues/9536 + dnsConfig: + nameservers: + - 127.0.0.53 + options: + - name: edns0 + - name: trust-ad + kubernetesServiceEndpoint: + host: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + port: "{{ .Cluster.spec.controlPlaneEndpoint.port }}" + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: metrics-server-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + metrics-server: enabled + resources: + - kind: ConfigMap + name: metrics-server-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + metrics-server: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader + rules: + - apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - emptyDir: {} + name: tmp-dir + --- + apiVersion: apiregistration.k8s.io/v1 + kind: APIService + metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io + spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: metrics-server-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/prow-azl4/kustomization.yaml b/templates/test/ci/prow-azl4/kustomization.yaml new file mode 100644 index 00000000000..a0004c72287 --- /dev/null +++ b/templates/test/ci/prow-azl4/kustomization.yaml @@ -0,0 +1,36 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../../../flavors/default +- ../../../addons/cluster-api-helm/calico.yaml +- ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml +patches: +- path: ../patches/tags.yaml +- path: ../patches/mhc.yaml +- path: ../patches/controller-manager.yaml +- path: ../patches/uami-md-0.yaml +- path: ../patches/uami-control-plane.yaml +- path: ../patches/cluster-label-calico.yaml +- path: ../patches/cluster-label-cloud-provider-azure.yaml +- path: patches/controller-manager.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 +- path: patches/kubeadm-config-template-azl4.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + version: v1beta1 +- path: patches/azuremachinetemplate-azl4-image.yaml +- path: patches/cloud-provider-azure-cacertdir.yaml +- path: patches/cloud-provider-azure-ci-cacertdir.yaml + +sortOptions: + order: fifo diff --git a/templates/test/ci/prow-azl4/patches/azuremachinetemplate-azl4-image.yaml b/templates/test/ci/prow-azl4/patches/azuremachinetemplate-azl4-image.yaml new file mode 100644 index 00000000000..59bc48dd5b6 --- /dev/null +++ b/templates/test/ci/prow-azl4/patches/azuremachinetemplate-azl4-image.yaml @@ -0,0 +1,25 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-4-gen2 + version: ${AZL4_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-4-gen2 + version: ${AZL4_VERSION} diff --git a/templates/test/ci/prow-azl4/patches/cloud-provider-azure-cacertdir.yaml b/templates/test/ci/prow-azl4/patches/cloud-provider-azure-cacertdir.yaml new file mode 100644 index 00000000000..1a19310be5e --- /dev/null +++ b/templates/test/ci/prow-azl4/patches/cloud-provider-azure-cacertdir.yaml @@ -0,0 +1,12 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 diff --git a/templates/test/ci/prow-azl4/patches/cloud-provider-azure-ci-cacertdir.yaml b/templates/test/ci/prow-azl4/patches/cloud-provider-azure-ci-cacertdir.yaml new file mode 100644 index 00000000000..cf26adf88c4 --- /dev/null +++ b/templates/test/ci/prow-azl4/patches/cloud-provider-azure-ci-cacertdir.yaml @@ -0,0 +1,23 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" diff --git a/templates/test/ci/prow-azl4/patches/controller-manager.yaml b/templates/test/ci/prow-azl4/patches/controller-manager.yaml new file mode 100644 index 00000000000..75ffb586d65 --- /dev/null +++ b/templates/test/ci/prow-azl4/patches/controller-manager.yaml @@ -0,0 +1,46 @@ +- op: add + path: /spec/kubeadmConfigSpec/files/0 + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Allow Azure service IP addresses (required for Azure resources) + iptables -A INPUT -s 168.63.129.16 -j ACCEPT + iptables -A OUTPUT -d 168.63.129.16 -j ACCEPT + + # Kubernetes API Server (port 6443) - bound to all IPv6 interfaces, needs external access + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + + # etcd server communication + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + + # Allow traffic to Kubernetes service network (10.96.0.0/12) + iptables -A OUTPUT -d 10.96.0.0/12 -j ACCEPT + iptables -A INPUT -s 10.96.0.0/12 -j ACCEPT + + # Allow traffic to/from node network (10.1.0.0/24) + iptables -A OUTPUT -d 10.1.0.0/24 -j ACCEPT + iptables -A INPUT -s 10.1.0.0/24 -j ACCEPT + + # Allow traffic to/from Calico pod network + iptables -A OUTPUT -d 192.168.0.0/24 -j ACCEPT + iptables -A INPUT -s 192.168.0.0/24 -j ACCEPT + + # Save the rules following Azure Linux 4 approach + iptables-save > /etc/systemd/scripts/ip4save + path: /tmp/azl4-setup.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/kubeadmConfigSpec/preKubeadmCommands/0 + value: + bash -c /tmp/azl4-setup.sh diff --git a/templates/test/ci/prow-azl4/patches/disable-vm-bootstrap-extension.yaml b/templates/test/ci/prow-azl4/patches/disable-vm-bootstrap-extension.yaml new file mode 100644 index 00000000000..5d06409a84d --- /dev/null +++ b/templates/test/ci/prow-azl4/patches/disable-vm-bootstrap-extension.yaml @@ -0,0 +1,17 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + disableVMBootstrapExtension: true +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + disableVMBootstrapExtension: true diff --git a/templates/test/ci/prow-azl4/patches/kubeadm-config-template-azl4.yaml b/templates/test/ci/prow-azl4/patches/kubeadm-config-template-azl4.yaml new file mode 100644 index 00000000000..3bd66c82906 --- /dev/null +++ b/templates/test/ci/prow-azl4/patches/kubeadm-config-template-azl4.yaml @@ -0,0 +1,63 @@ +- op: add + path: /spec/template/spec/files/0 + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Allow Azure service IP addresses (required for Azure resources) + iptables -A INPUT -s 168.63.129.16 -j ACCEPT + iptables -A OUTPUT -d 168.63.129.16 -j ACCEPT + + # Allow localhost traffic + iptables -A INPUT -i lo -j ACCEPT + iptables -A OUTPUT -o lo -j ACCEPT + + # Allow established and related connections + iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT + iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT + + # SSH (port 22) + # iptables -A INPUT -p tcp --dport 22 -j ACCEPT + + # Kubelet API (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + + # Allow traffic to Kubernetes service network (10.96.0.0/12) + iptables -A OUTPUT -d 10.96.0.0/12 -j ACCEPT + iptables -A INPUT -s 10.96.0.0/12 -j ACCEPT + + # Allow traffic to/from Calico pod network (192.168.0.0/16) + iptables -A OUTPUT -d 192.168.0.0/16 -j ACCEPT + iptables -A INPUT -s 192.168.0.0/16 -j ACCEPT + + # Allow traffic to/from node network (10.1.0.0/24) + iptables -A OUTPUT -d 10.1.0.0/24 -j ACCEPT + iptables -A INPUT -s 10.1.0.0/24 -j ACCEPT + + # Calico networking requirements + # Calico Typha (port 5473) + iptables -A INPUT -p tcp --dport 5473 -j ACCEPT + + # VXLAN for overlay networking (port 4789 UDP) + iptables -A INPUT -p udp --dport 4789 -j ACCEPT + + # BGP for node-to-node communication (port 179) + iptables -A INPUT -p tcp --d port 179 -j ACCEPT + + # DNS (port 53) + iptables -A INPUT -p udp --dport 53 -j ACCEPT + iptables -A OUTPUT -p udp --dport 53 -j ACCEPT + + # Save the rules following Azure Linux 4 approach + iptables-save > /etc/systemd/scripts/ip4save + path: /tmp/azl4-setup.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/0 + value: + bash -c /tmp/azl4-setup.sh diff --git a/templates/test/ci/prow-azl4/patches/remove-marketplace-image.yaml b/templates/test/ci/prow-azl4/patches/remove-marketplace-image.yaml new file mode 100644 index 00000000000..c44ae2bdf32 --- /dev/null +++ b/templates/test/ci/prow-azl4/patches/remove-marketplace-image.yaml @@ -0,0 +1,19 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + image: + marketplace: null +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: + marketplace: null diff --git a/templates/test/ci/prow-ci-version-azl4/kustomization.yaml b/templates/test/ci/prow-ci-version-azl4/kustomization.yaml new file mode 100644 index 00000000000..3d2679f1058 --- /dev/null +++ b/templates/test/ci/prow-ci-version-azl4/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../prow-ci-version +patches: +- path: ../prow-azl4/patches/controller-manager.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 +- path: ../prow-azl4/patches/kubeadm-config-template-azl4.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + version: v1beta1 +- path: ../prow-azl4/patches/azuremachinetemplate-azl4-image.yaml +- path: ../prow-azl4/patches/remove-marketplace-image.yaml +- path: ../prow-azl4/patches/cloud-provider-azure-cacertdir.yaml +- path: ../prow-azl4/patches/cloud-provider-azure-ci-cacertdir.yaml + +sortOptions: + order: fifo diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 1df14cba8c1..66f0da47c07 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -1377,6 +1377,95 @@ spec: }) }) + Context("Creating a highly-available cluster with Azure Linux 4 [OPTIONAL]", func() { + It("with three controlplane node and two worker nodes", func() { + clusterName = getClusterName(clusterNamePrefix, "azl4") + kubernetesVersion := e2eConfig.MustGetVariable(capi_e2e.KubernetesVersion) + kubernetesVersion = strings.TrimPrefix(kubernetesVersion, "v") + Expect(os.Setenv("AZL4_VERSION", kubernetesVersion)).To(Succeed()) + skipResourceGroupCheck = true + + clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( + specName, + withNamespace(namespace.Name), + withClusterName(clusterName), + withFlavor("azl4"), + withControlPlaneMachineCount(3), + withWorkerMachineCount(2), + withControlPlaneInterval(specName, "wait-control-plane-ha"), + withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ + WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + }), + withPostMachinesProvisioned(func() { + EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { + return DaemonsetsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }), + ), result) + + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + + By("Verifying security rules are deleted on azure side", func() { + AzureSecurityGroupsSpec(ctx, func() AzureSecurityGroupsSpecInput { + return AzureSecurityGroupsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + Cluster: result.Cluster, + WaitForUpdate: e2eConfig.GetIntervals(specName, "wait-nsg-update"), + } + }) + }) + + By("Validating failure domains", func() { + AzureFailureDomainsSpec(ctx, func() AzureFailureDomainsSpecInput { + return AzureFailureDomainsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Cluster: result.Cluster, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + + By("Creating an accessible load balancer", func() { + AzureLBSpec(ctx, func() AzureLBSpecInput { + return AzureLBSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } + }) + }) + + By("Validating network policies", func() { + AzureNetPolSpec(ctx, func() AzureNetPolSpecInput { + return AzureNetPolSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } + }) + }) + + By("PASSED!") + }) + }) + // TODO: add a same test as above for a windows cluster Context("Testing autoscaling from zero with Cluster Autoscaler [REQUIRED]", func() { diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index a153528a897..11358a7dbb4 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -191,6 +191,8 @@ providers: targetName: "cluster-template-conformance-ci-artifacts-dra.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml" targetName: "cluster-template-conformance-ci-artifacts-azl3.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-azl4.yaml" + targetName: "cluster-template-conformance-ci-artifacts-azl4.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml" targetName: "cluster-template-machine-pool-flex.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks.yaml" @@ -231,6 +233,8 @@ providers: targetName: "cluster-template-dalec-custom-builds.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-azl3.yaml" targetName: "cluster-template-azl3.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-azl4.yaml" + targetName: "cluster-template-azl4.yaml" replacements: - old: "--v=0" new: "--v=2" From e4868e73957ca690c9bec97453416c4eef7169fb Mon Sep 17 00:00:00 2001 From: Matt Boersma Date: Thu, 14 May 2026 15:08:07 -0600 Subject: [PATCH 2/2] TEMP: skip other [OPTIONAL] e2e tests to isolate Azure Linux 4 spec --- test/e2e/azure_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 66f0da47c07..ca22e1be271 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -144,7 +144,7 @@ var _ = Describe("Workload cluster creation", func() { if os.Getenv("USE_LOCAL_KIND_REGISTRY") != "true" { // This spec expects a user-assigned identity with Contributor role assignment named "cloud-provider-user-identity" in a "capz-ci" // resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables. - Context("Creating a private cluster [OPTIONAL]", func() { + PContext("Creating a private cluster [OPTIONAL]", func() { It("Creates a public management cluster in a custom vnet", func() { clusterName = getClusterName(clusterNamePrefix, "public-custom-vnet") By("Creating a custom virtual network", func() { @@ -347,7 +347,7 @@ var _ = Describe("Workload cluster creation", func() { }) }) - Context("Creating a Flatcar sysext cluster [OPTIONAL]", func() { + PContext("Creating a Flatcar sysext cluster [OPTIONAL]", func() { It("With Flatcar control-plane and worker nodes", func() { clusterName = getClusterName(clusterNamePrefix, "flatcar-sysext") clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( @@ -385,7 +385,7 @@ var _ = Describe("Workload cluster creation", func() { }) }) - Context("Creating a cluster with spot vms [OPTIONAL]", func() { + PContext("Creating a cluster with spot vms [OPTIONAL]", func() { It("With spot vm machine deployments", func() { clusterName = getClusterName(clusterNamePrefix, "spot") clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( @@ -535,7 +535,7 @@ var _ = Describe("Workload cluster creation", func() { // You can override the default SKU `Standard_NV12s_v3` and `Premium_LRS` storage by setting // the `AZURE_GPU_NODE_MACHINE_TYPE` and `AZURE_GPU_NODE_STORAGE_TYPE` environment variables. // See https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/ for pricing. - Context("Creating a GPU-enabled cluster [OPTIONAL]", func() { + PContext("Creating a GPU-enabled cluster [OPTIONAL]", func() { It("with a single control plane node and 1 node", func() { Skip("Skipping since the e2e subscription has no quota for GPU SKUs") clusterName = getClusterName(clusterNamePrefix, "gpu") @@ -594,7 +594,7 @@ var _ = Describe("Workload cluster creation", func() { }) // ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`. - Context("Creating a cluster with VMSS flex machinepools [OPTIONAL]", func() { + PContext("Creating a cluster with VMSS flex machinepools [OPTIONAL]", func() { It("with 1 control plane node and 1 machinepool", func() { clusterName = getClusterName(clusterNamePrefix, "flex") clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( @@ -938,7 +938,7 @@ var _ = Describe("Workload cluster creation", func() { // ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`. // This spec expects a user-assigned identity named "cloud-provider-user-identity" in a "capz-ci" // resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables. - Context("Creating a dual-stack cluster [OPTIONAL]", func() { + PContext("Creating a dual-stack cluster [OPTIONAL]", func() { It("With dual-stack worker node", func() { By("using user-assigned identity") clusterName = getClusterName(clusterNamePrefix, "dual-stack") @@ -1006,7 +1006,7 @@ var _ = Describe("Workload cluster creation", func() { }) }) - Context("Creating clusters using clusterclass [OPTIONAL]", func() { + PContext("Creating clusters using clusterclass [OPTIONAL]", func() { It("with a single control plane node and one linux worker node", func() { // Use ci-default as the clusterclass name so test infra can find the clusterclass template Expect(os.Setenv("CLUSTER_CLASS_NAME", "ci-default")).To(Succeed()) @@ -1050,7 +1050,7 @@ var _ = Describe("Workload cluster creation", func() { }) }) - Context("Creating RKE2 clusters using clusterclass [OPTIONAL]", func() { + PContext("Creating RKE2 clusters using clusterclass [OPTIONAL]", func() { It("with 3 control plane node and one linux worker node", func() { // Use ci-rke2 as the clusterclass name so test infra can find the clusterclass template Expect(os.Setenv("CLUSTER_CLASS_NAME", "ci-rke2")).To(Succeed()) @@ -1150,7 +1150,7 @@ spec: // resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables. // You can also override the default SKU `Standard_DS2_v2` and `Standard_DS4_v2` storage by setting // the `AZURE_EDGEZONE_CONTROL_PLANE_MACHINE_TYPE` and `AZURE_EDGEZONE_NODE_MACHINE_TYPE` environment variables. - Context("Creating clusters on public MEC [OPTIONAL]", func() { + PContext("Creating clusters on public MEC [OPTIONAL]", func() { It("with 1 control plane nodes and 1 worker node", func() { Skip("Skipping public MEC test until a new edgezone is available") By("using user-assigned identity") @@ -1191,7 +1191,7 @@ spec: }) }) - Context("Creating a self-managed VM based cluster using API Server ILB feature gate using default template [OPTIONAL][API-Server-ILB]", func() { + PContext("Creating a self-managed VM based cluster using API Server ILB feature gate using default template [OPTIONAL][API-Server-ILB]", func() { It("with three controlplane node and three worker nodes", func() { clusterName = getClusterName(clusterNamePrefix, "apiserver-ilb") @@ -1237,7 +1237,7 @@ spec: }) }) - Context("Creating a self-managed VM based cluster using API Server ILB feature gate and fully spec-ed out APIServer ILB template [OPTIONAL][API-Server-ILB]", func() { + PContext("Creating a self-managed VM based cluster using API Server ILB feature gate and fully spec-ed out APIServer ILB template [OPTIONAL][API-Server-ILB]", func() { It("with three controlplane node and three worker nodes", func() { clusterName = getClusterName(clusterNamePrefix, "apiserver-ilb") @@ -1288,7 +1288,7 @@ spec: }) }) - Context("Creating a highly-available cluster with Azure Linux 3 [OPTIONAL]", func() { + PContext("Creating a highly-available cluster with Azure Linux 3 [OPTIONAL]", func() { It("with three controlplane node and two worker nodes", func() { clusterName = getClusterName(clusterNamePrefix, "azl3") kubernetesVersion := e2eConfig.MustGetVariable(capi_e2e.KubernetesVersion)