From bd4a7c1427f1c0e9112d4df8a99e2849234b6b57 Mon Sep 17 00:00:00 2001 From: Rahul Ganesh Date: Tue, 12 May 2026 16:50:37 -0700 Subject: [PATCH] Fix registry mirror hosts.toml double /v2/ regression from containerd v2 migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit URLs are used as-is) to hosts.toml format (where containerd auto-appends /v2/ to the host path). Since ToAPIEndpoints() already prepends /v2/ before the namespace, containerd produced a broken double /v2/ in requests: /v2//v2//manifests/ → 404. Fix: Always set override_path = true in hosts.toml entries so containerd uses the host path as-is without auto-appending /v2/. Also update ToAPIEndpoint to always produce a /v2 path prefix (even for pathless URLs) ensuring all entries have a well-formed OCI API root. Signed-off-by: Rahul Ganesh --- internal/test/registrymirror.go | 12 ++++--- pkg/clusterapi/config/hosts.toml | 1 + pkg/clusterapi/registry_mirror.go | 2 +- pkg/clusterapi/registry_mirror_test.go | 25 +++++++++----- pkg/executables/config/hosts.toml | 1 + pkg/executables/kind.go | 2 +- .../hosts_toml_insecure_public_ecr_aws.toml | 1 + .../hosts_toml_insecure_registry_mirror.toml | 3 +- ...s_toml_with_auth_783794618700_dkr_ecr.toml | 1 + .../hosts_toml_with_auth_public_ecr_aws.toml | 1 + .../hosts_toml_with_auth_registry_mirror.toml | 5 +-- .../hosts_toml_with_ca_public_ecr_aws.toml | 3 +- .../hosts_toml_with_ca_registry_mirror.toml | 3 +- .../cloudstack/config/template-cp.yaml | 8 +++-- .../cloudstack/config/template-md.yaml | 8 +++-- pkg/providers/cloudstack/template.go | 2 ++ pkg/providers/docker/config/template-cp.yaml | 10 +++--- pkg/providers/docker/config/template-md.yaml | 10 +++--- pkg/providers/docker/docker.go | 1 + pkg/providers/nutanix/config/cp-template.yaml | 10 +++--- pkg/providers/nutanix/config/md-template.yaml | 10 +++--- pkg/providers/nutanix/template.go | 2 ++ .../expected_results_registry_mirror.yaml | 10 +++--- .../expected_results_registry_mirror_md.yaml | 10 +++--- pkg/providers/snow/apibuilder_test.go | 33 ++++++++++++------- .../tinkerbell/config/template-cp.yaml | 10 +++--- .../tinkerbell/config/template-md.yaml | 10 +++--- pkg/providers/tinkerbell/controlplane_test.go | 14 ++++---- pkg/providers/tinkerbell/template.go | 1 + pkg/providers/vsphere/config/template-cp.yaml | 10 +++--- pkg/providers/vsphere/config/template-md.yaml | 10 +++--- pkg/providers/vsphere/template.go | 2 ++ pkg/registrymirror/containerd/utils.go | 2 ++ pkg/registrymirror/containerd/utils_test.go | 6 ++-- 34 files changed, 153 insertions(+), 86 deletions(-) diff --git a/internal/test/registrymirror.go b/internal/test/registrymirror.go index d2f1016f70d0..f747fe1e98ce 100644 --- a/internal/test/registrymirror.go +++ b/internal/test/registrymirror.go @@ -51,8 +51,9 @@ func RegistryMirrorConfigFilesInsecureSkipVerify() []bootstrapv1beta2.File { { Content: `server = "https://0.0.0.0:5000" -[host."https://0.0.0.0:5000"] +[host."https://0.0.0.0:5000/v2"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true `, Owner: "root:root", @@ -61,8 +62,9 @@ func RegistryMirrorConfigFilesInsecureSkipVerify() []bootstrapv1beta2.File { { Content: `server = "https://public.ecr.aws" -[host."https://0.0.0.0:5000"] +[host."https://0.0.0.0:5000/v2"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true `, Owner: "root:root", @@ -90,8 +92,9 @@ func RegistryMirrorConfigFilesInsecureSkipVerifyAndCACert() []bootstrapv1beta2.F { Content: `server = "https://0.0.0.0:5000" -[host."https://0.0.0.0:5000"] +[host."https://0.0.0.0:5000/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/0.0.0.0:5000/ca.crt" skip_verify = true `, @@ -101,8 +104,9 @@ func RegistryMirrorConfigFilesInsecureSkipVerifyAndCACert() []bootstrapv1beta2.F { Content: `server = "https://public.ecr.aws" -[host."https://0.0.0.0:5000"] +[host."https://0.0.0.0:5000/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/0.0.0.0:5000/ca.crt" skip_verify = true `, diff --git a/pkg/clusterapi/config/hosts.toml b/pkg/clusterapi/config/hosts.toml index f72e8c9c713a..6e8b86d2ce80 100644 --- a/pkg/clusterapi/config/hosts.toml +++ b/pkg/clusterapi/config/hosts.toml @@ -2,6 +2,7 @@ server = "https://{{ .server }}" [host."https://{{ .host }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" {{- end }} diff --git a/pkg/clusterapi/registry_mirror.go b/pkg/clusterapi/registry_mirror.go index 2b2751441e67..c4a2475a1bec 100644 --- a/pkg/clusterapi/registry_mirror.go +++ b/pkg/clusterapi/registry_mirror.go @@ -129,7 +129,7 @@ func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfigura } // Mirror base hosts.toml - mirrorBaseContent, err := hostsFileContent(registryMirror, registryMirror.BaseRegistry, registryMirror.BaseRegistry) + mirrorBaseContent, err := hostsFileContent(registryMirror, registryMirror.BaseRegistry, containerd.ToAPIEndpoint(registryMirror.BaseRegistry)) if err != nil { return nil, err } diff --git a/pkg/clusterapi/registry_mirror_test.go b/pkg/clusterapi/registry_mirror_test.go index dbd639a47519..679323590ee0 100644 --- a/pkg/clusterapi/registry_mirror_test.go +++ b/pkg/clusterapi/registry_mirror_test.go @@ -58,8 +58,9 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://1.2.3.4:443" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" `, }, @@ -70,6 +71,7 @@ var registryMirrorTests = []struct { [host."https://1.2.3.4:443/v2/curated-packages"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" `, }, @@ -80,6 +82,7 @@ var registryMirrorTests = []struct { [host."https://1.2.3.4:443/v2/eks-anywhere"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" `, }, @@ -113,8 +116,9 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://1.2.3.4:443" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true `, }, @@ -123,17 +127,18 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://public.ecr.aws" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true `, }, }, wantRegistryConfig: bootstrapv1beta2.RegistryMirrorConfiguration{ - Endpoint: "1.2.3.4:443", + Endpoint: "1.2.3.4:443/v2", }, wantRegistryConfigEtcd: &etcdbootstrapv1.RegistryMirrorConfiguration{ - Endpoint: "1.2.3.4:443", + Endpoint: "1.2.3.4:443/v2", }, }, { @@ -162,8 +167,9 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://1.2.3.4:443" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" skip_verify = true `, @@ -173,19 +179,20 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://public.ecr.aws" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" skip_verify = true `, }, }, wantRegistryConfig: bootstrapv1beta2.RegistryMirrorConfiguration{ - Endpoint: "1.2.3.4:443", + Endpoint: "1.2.3.4:443/v2", CACert: "xyz", }, wantRegistryConfigEtcd: &etcdbootstrapv1.RegistryMirrorConfiguration{ - Endpoint: "1.2.3.4:443", + Endpoint: "1.2.3.4:443/v2", CACert: "xyz", }, }, diff --git a/pkg/executables/config/hosts.toml b/pkg/executables/config/hosts.toml index 310e3503d63c..f7401871ded4 100644 --- a/pkg/executables/config/hosts.toml +++ b/pkg/executables/config/hosts.toml @@ -2,6 +2,7 @@ server = "https://{{.Server}}" [host."https://{{.Host}}"] capabilities = ["pull", "resolve"] + override_path = true {{- if .CACertPath }} ca = "{{.CACertPath}}" {{- else }} diff --git a/pkg/executables/kind.go b/pkg/executables/kind.go index 0b4283994951..e4f79ee1366e 100644 --- a/pkg/executables/kind.go +++ b/pkg/executables/kind.go @@ -311,7 +311,7 @@ func (k *Kind) setupRegistryMirror(clusterSpec *cluster.Spec, registryMirror *re // Setup configuration for the mirror registry err := setupRegistryConfig(RegistryConfig{ Server: mirrorBase, - Host: mirrorBase, + Host: containerd.ToAPIEndpoint(mirrorBase), CACertPath: mountedCACertPath, AuthHeader: authHeader, OutputDir: filepath.Join(certsBasePath, mirrorBase), diff --git a/pkg/executables/testdata/hosts_toml_insecure_public_ecr_aws.toml b/pkg/executables/testdata/hosts_toml_insecure_public_ecr_aws.toml index ef0169d1c525..8e4f5e7ba46f 100644 --- a/pkg/executables/testdata/hosts_toml_insecure_public_ecr_aws.toml +++ b/pkg/executables/testdata/hosts_toml_insecure_public_ecr_aws.toml @@ -2,4 +2,5 @@ server = "https://public.ecr.aws" [host."https://registry-mirror.test:443/v2/eks-anywhere"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true diff --git a/pkg/executables/testdata/hosts_toml_insecure_registry_mirror.toml b/pkg/executables/testdata/hosts_toml_insecure_registry_mirror.toml index 288c56da1917..a16dc75e30d5 100644 --- a/pkg/executables/testdata/hosts_toml_insecure_registry_mirror.toml +++ b/pkg/executables/testdata/hosts_toml_insecure_registry_mirror.toml @@ -1,5 +1,6 @@ server = "https://registry-mirror.test:443" -[host."https://registry-mirror.test:443"] +[host."https://registry-mirror.test:443/v2"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true diff --git a/pkg/executables/testdata/hosts_toml_with_auth_783794618700_dkr_ecr.toml b/pkg/executables/testdata/hosts_toml_with_auth_783794618700_dkr_ecr.toml index fe59a99cea20..60c8e12a996f 100644 --- a/pkg/executables/testdata/hosts_toml_with_auth_783794618700_dkr_ecr.toml +++ b/pkg/executables/testdata/hosts_toml_with_auth_783794618700_dkr_ecr.toml @@ -2,6 +2,7 @@ server = "https://783794618700.dkr.ecr.us-west-2.amazonaws.com" [host."https://registry-mirror.test:443/v2/curated-packages"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true [host."https://registry-mirror.test:443/v2/curated-packages".header] authorization = "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" diff --git a/pkg/executables/testdata/hosts_toml_with_auth_public_ecr_aws.toml b/pkg/executables/testdata/hosts_toml_with_auth_public_ecr_aws.toml index 83b9d9a15529..99b3a04636b4 100644 --- a/pkg/executables/testdata/hosts_toml_with_auth_public_ecr_aws.toml +++ b/pkg/executables/testdata/hosts_toml_with_auth_public_ecr_aws.toml @@ -2,6 +2,7 @@ server = "https://public.ecr.aws" [host."https://registry-mirror.test:443/v2/eks-anywhere"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true [host."https://registry-mirror.test:443/v2/eks-anywhere".header] authorization = "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" diff --git a/pkg/executables/testdata/hosts_toml_with_auth_registry_mirror.toml b/pkg/executables/testdata/hosts_toml_with_auth_registry_mirror.toml index 2df80ebe0e00..b17bf568ccfd 100644 --- a/pkg/executables/testdata/hosts_toml_with_auth_registry_mirror.toml +++ b/pkg/executables/testdata/hosts_toml_with_auth_registry_mirror.toml @@ -1,7 +1,8 @@ server = "https://registry-mirror.test:443" -[host."https://registry-mirror.test:443"] +[host."https://registry-mirror.test:443/v2"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true - [host."https://registry-mirror.test:443".header] + [host."https://registry-mirror.test:443/v2".header] authorization = "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" diff --git a/pkg/executables/testdata/hosts_toml_with_ca_public_ecr_aws.toml b/pkg/executables/testdata/hosts_toml_with_ca_public_ecr_aws.toml index 21d392367e1e..2db0775988ae 100644 --- a/pkg/executables/testdata/hosts_toml_with_ca_public_ecr_aws.toml +++ b/pkg/executables/testdata/hosts_toml_with_ca_public_ecr_aws.toml @@ -1,5 +1,6 @@ server = "https://public.ecr.aws" -[host."https://registry-mirror.test:443"] +[host."https://registry-mirror.test:443/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/registry-mirror.test:443/ca.crt" diff --git a/pkg/executables/testdata/hosts_toml_with_ca_registry_mirror.toml b/pkg/executables/testdata/hosts_toml_with_ca_registry_mirror.toml index 9fbcac3eac70..357ba927e859 100644 --- a/pkg/executables/testdata/hosts_toml_with_ca_registry_mirror.toml +++ b/pkg/executables/testdata/hosts_toml_with_ca_registry_mirror.toml @@ -1,5 +1,6 @@ server = "https://registry-mirror.test:443" -[host."https://registry-mirror.test:443"] +[host."https://registry-mirror.test:443/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/registry-mirror.test:443/ca.crt" diff --git a/pkg/providers/cloudstack/config/template-cp.yaml b/pkg/providers/cloudstack/config/template-cp.yaml index c41bdc1ae4dc..338cf5f323cf 100644 --- a/pkg/providers/cloudstack/config/template-cp.yaml +++ b/pkg/providers/cloudstack/config/template-cp.yaml @@ -284,9 +284,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -300,9 +301,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/cloudstack/config/template-md.yaml b/pkg/providers/cloudstack/config/template-md.yaml index 09b30aa29a6c..a326c9bb004d 100644 --- a/pkg/providers/cloudstack/config/template-md.yaml +++ b/pkg/providers/cloudstack/config/template-md.yaml @@ -74,9 +74,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -90,9 +91,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/cloudstack/template.go b/pkg/providers/cloudstack/template.go index 59616a05f4ef..b39151161f2d 100644 --- a/pkg/providers/cloudstack/template.go +++ b/pkg/providers/cloudstack/template.go @@ -224,6 +224,7 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) values["mirrorBase"] = registryMirror.BaseRegistry + values["mirrorBaseAPIEndpoint"] = containerd.ToAPIEndpoint(registryMirror.BaseRegistry) values["insecureSkip"] = registryMirror.InsecureSkipVerify values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror()) if len(registryMirror.CACertContent) > 0 { @@ -408,6 +409,7 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupConfiguration registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) values["mirrorBase"] = registryMirror.BaseRegistry + values["mirrorBaseAPIEndpoint"] = containerd.ToAPIEndpoint(registryMirror.BaseRegistry) values["insecureSkip"] = registryMirror.InsecureSkipVerify if len(registryMirror.CACertContent) > 0 { values["registryCACert"] = registryMirror.CACertContent diff --git a/pkg/providers/docker/config/template-cp.yaml b/pkg/providers/docker/config/template-cp.yaml index 72434041c82a..ddf579061963 100644 --- a/pkg/providers/docker/config/template-cp.yaml +++ b/pkg/providers/docker/config/template-cp.yaml @@ -186,9 +186,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -198,7 +199,7 @@ spec: {{- end }} {{- end }} {{- if .registryAuth }} - [host."https://{{ .mirrorBase }}".header] + [host."https://{{ .mirrorBaseAPIEndpoint }}".header] authorization = "Basic {{ printf "%s:%s" .registryUsername .registryPassword | b64enc }}" {{- end }} owner: root:root @@ -206,9 +207,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/docker/config/template-md.yaml b/pkg/providers/docker/config/template-md.yaml index 401d7f1899b0..321ddb52544b 100644 --- a/pkg/providers/docker/config/template-md.yaml +++ b/pkg/providers/docker/config/template-md.yaml @@ -60,9 +60,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -72,7 +73,7 @@ spec: {{- end }} {{- end }} {{- if .registryAuth }} - [host."https://{{ .mirrorBase }}".header] + [host."https://{{ .mirrorBaseAPIEndpoint }}".header] authorization = "Basic {{ printf "%s:%s" .registryUsername .registryPassword | b64enc }}" {{- end }} owner: root:root @@ -80,9 +81,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/docker/docker.go b/pkg/providers/docker/docker.go index 02cac15880c7..6f4abc91c8b1 100644 --- a/pkg/providers/docker/docker.go +++ b/pkg/providers/docker/docker.go @@ -672,6 +672,7 @@ func populateRegistryMirrorValues(clusterSpec *cluster.Spec, values map[string]i registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) values["mirrorBase"] = registryMirror.BaseRegistry + values["mirrorBaseAPIEndpoint"] = containerd.ToAPIEndpoint(registryMirror.BaseRegistry) values["insecureSkip"] = registryMirror.InsecureSkipVerify values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror()) if len(registryMirror.CACertContent) > 0 { diff --git a/pkg/providers/nutanix/config/cp-template.yaml b/pkg/providers/nutanix/config/cp-template.yaml index e633b3ba3f44..86b8965da9a6 100644 --- a/pkg/providers/nutanix/config/cp-template.yaml +++ b/pkg/providers/nutanix/config/cp-template.yaml @@ -310,9 +310,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -322,7 +323,7 @@ spec: {{- end }} {{- end }} {{- if .registryAuth }} - [host."https://{{ .mirrorBase }}".header] + [host."https://{{ .mirrorBaseAPIEndpoint }}".header] authorization = "Basic {{ printf "%s:%s" .registryUsername .registryPassword | b64enc }}" {{- end }} owner: root:root @@ -330,9 +331,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/nutanix/config/md-template.yaml b/pkg/providers/nutanix/config/md-template.yaml index f5d4bd953c93..d438a6c0bf82 100644 --- a/pkg/providers/nutanix/config/md-template.yaml +++ b/pkg/providers/nutanix/config/md-template.yaml @@ -308,9 +308,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -320,7 +321,7 @@ spec: {{- end }} {{- end }} {{- if .registryAuth }} - [host."https://{{ .mirrorBase }}".header] + [host."https://{{ .mirrorBaseAPIEndpoint }}".header] authorization = "Basic {{ printf "%s:%s" .registryUsername .registryPassword | b64enc }}" {{- end }} owner: root:root @@ -328,9 +329,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/nutanix/template.go b/pkg/providers/nutanix/template.go index 8403e786df2d..5c6281db7dd6 100644 --- a/pkg/providers/nutanix/template.go +++ b/pkg/providers/nutanix/template.go @@ -262,6 +262,7 @@ func buildTemplateMapCP( registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) values["mirrorBase"] = registryMirror.BaseRegistry + values["mirrorBaseAPIEndpoint"] = containerd.ToAPIEndpoint(registryMirror.BaseRegistry) values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror()) values["insecureSkip"] = registryMirror.InsecureSkipVerify if len(registryMirror.CACertContent) > 0 { @@ -453,6 +454,7 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupMachineSpec v1 registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) values["mirrorBase"] = registryMirror.BaseRegistry + values["mirrorBaseAPIEndpoint"] = containerd.ToAPIEndpoint(registryMirror.BaseRegistry) values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror()) values["insecureSkip"] = registryMirror.InsecureSkipVerify if len(registryMirror.CACertContent) > 0 { diff --git a/pkg/providers/nutanix/testdata/expected_results_registry_mirror.yaml b/pkg/providers/nutanix/testdata/expected_results_registry_mirror.yaml index fe720863bfbe..c984dfbed55e 100644 --- a/pkg/providers/nutanix/testdata/expected_results_registry_mirror.yaml +++ b/pkg/providers/nutanix/testdata/expected_results_registry_mirror.yaml @@ -195,20 +195,22 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://1.2.3.4:1234" - - [host."https://1.2.3.4:1234"] + + [host."https://1.2.3.4:1234/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" skip_verify = true - [host."https://1.2.3.4:1234".header] + [host."https://1.2.3.4:1234/v2".header] authorization = "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" owner: root:root path: "/etc/containerd/certs.d/1.2.3.4:1234/hosts.toml" - content: | server = "https://public.ecr.aws" - + [host."https://1.2.3.4:1234/v2/eks-anywhere"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" skip_verify = true [host."https://1.2.3.4:1234/v2/eks-anywhere".header] diff --git a/pkg/providers/nutanix/testdata/expected_results_registry_mirror_md.yaml b/pkg/providers/nutanix/testdata/expected_results_registry_mirror_md.yaml index 712c74b665d2..b86013d08936 100644 --- a/pkg/providers/nutanix/testdata/expected_results_registry_mirror_md.yaml +++ b/pkg/providers/nutanix/testdata/expected_results_registry_mirror_md.yaml @@ -112,20 +112,22 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://1.2.3.4:1234" - - [host."https://1.2.3.4:1234"] + + [host."https://1.2.3.4:1234/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" skip_verify = true - [host."https://1.2.3.4:1234".header] + [host."https://1.2.3.4:1234/v2".header] authorization = "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" owner: root:root path: "/etc/containerd/certs.d/1.2.3.4:1234/hosts.toml" - content: | server = "https://public.ecr.aws" - + [host."https://1.2.3.4:1234/v2/eks-anywhere"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" skip_verify = true [host."https://1.2.3.4:1234/v2/eks-anywhere".header] diff --git a/pkg/providers/snow/apibuilder_test.go b/pkg/providers/snow/apibuilder_test.go index 81799a085184..95df1f21d685 100644 --- a/pkg/providers/snow/apibuilder_test.go +++ b/pkg/providers/snow/apibuilder_test.go @@ -285,8 +285,9 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://1.2.3.4:443" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" `, }, @@ -297,6 +298,7 @@ var registryMirrorTests = []struct { [host."https://1.2.3.4:443/v2/eks-anywhere"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" `, }, @@ -341,8 +343,9 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://1.2.3.4:443" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" `, }, @@ -353,6 +356,7 @@ var registryMirrorTests = []struct { [host."https://1.2.3.4:443/v2/curated-packages"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" `, }, @@ -363,6 +367,7 @@ var registryMirrorTests = []struct { [host."https://1.2.3.4:443/v2/eks-anywhere"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" `, }, @@ -392,8 +397,9 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://1.2.3.4:443" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true `, }, @@ -402,14 +408,15 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://public.ecr.aws" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true skip_verify = true `, }, }, wantRegistryConfig: bootstrapv1beta2.RegistryMirrorConfiguration{ - Endpoint: "1.2.3.4:443", + Endpoint: "1.2.3.4:443/v2", }, }, { @@ -431,8 +438,9 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://1.2.3.4:443" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true `, }, { @@ -440,13 +448,14 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://public.ecr.aws" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true `, }, }, wantRegistryConfig: bootstrapv1beta2.RegistryMirrorConfiguration{ - Endpoint: "1.2.3.4:443", + Endpoint: "1.2.3.4:443/v2", }, }, { @@ -475,8 +484,9 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://1.2.3.4:443" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" skip_verify = true `, @@ -486,15 +496,16 @@ var registryMirrorTests = []struct { Owner: "root:root", Content: `server = "https://public.ecr.aws" -[host."https://1.2.3.4:443"] +[host."https://1.2.3.4:443/v2"] capabilities = ["pull", "resolve"] + override_path = true ca = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" skip_verify = true `, }, }, wantRegistryConfig: bootstrapv1beta2.RegistryMirrorConfiguration{ - Endpoint: "1.2.3.4:443", + Endpoint: "1.2.3.4:443/v2", CACert: "xyz", }, }, diff --git a/pkg/providers/tinkerbell/config/template-cp.yaml b/pkg/providers/tinkerbell/config/template-cp.yaml index 370c3ce6bdbc..a4ed4c2ffd19 100644 --- a/pkg/providers/tinkerbell/config/template-cp.yaml +++ b/pkg/providers/tinkerbell/config/template-cp.yaml @@ -430,9 +430,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -442,7 +443,7 @@ spec: {{- end }} {{- end }} {{- if .registryAuth }} - [host."https://{{ .mirrorBase }}".header] + [host."https://{{ .mirrorBaseAPIEndpoint }}".header] authorization = "Basic {{ printf "%s:%s" .registryUsername .registryPassword | b64enc }}" {{- end }} owner: root:root @@ -450,9 +451,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/tinkerbell/config/template-md.yaml b/pkg/providers/tinkerbell/config/template-md.yaml index 9ec1c97f2fe8..a36b4acd33d7 100644 --- a/pkg/providers/tinkerbell/config/template-md.yaml +++ b/pkg/providers/tinkerbell/config/template-md.yaml @@ -238,9 +238,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -250,7 +251,7 @@ spec: {{- end }} {{- end }} {{- if .registryAuth }} - [host."https://{{ .mirrorBase }}".header] + [host."https://{{ .mirrorBaseAPIEndpoint }}".header] authorization = "Basic {{ printf "%s:%s" .registryUsername .registryPassword | b64enc }}" {{- end }} owner: root:root @@ -258,9 +259,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/tinkerbell/controlplane_test.go b/pkg/providers/tinkerbell/controlplane_test.go index 032d854e788c..ab4b1d939203 100644 --- a/pkg/providers/tinkerbell/controlplane_test.go +++ b/pkg/providers/tinkerbell/controlplane_test.go @@ -876,19 +876,21 @@ spec: path: /etc/containerd/config_append.toml - content: | server = "https://:" - - [host."https://:"] + + [host."https://:/v2"] capabilities = ["pull", "resolve"] - [host."https://:".header] + override_path = true + [host."https://:/v2".header] authorization = "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" owner: root:root path: /etc/containerd/certs.d/:/hosts.toml - content: | server = "https://public.ecr.aws" - - [host."https://:"] + + [host."https://:/v2"] capabilities = ["pull", "resolve"] - [host."https://:".header] + override_path = true + [host."https://:/v2".header] authorization = "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" owner: root:root path: /etc/containerd/certs.d/public.ecr.aws/hosts.toml diff --git a/pkg/providers/tinkerbell/template.go b/pkg/providers/tinkerbell/template.go index e195826783fe..08aebbcd55af 100644 --- a/pkg/providers/tinkerbell/template.go +++ b/pkg/providers/tinkerbell/template.go @@ -538,6 +538,7 @@ func populateRegistryMirrorValues(clusterSpec *cluster.Spec, values map[string]i registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) values["mirrorBase"] = registryMirror.BaseRegistry + values["mirrorBaseAPIEndpoint"] = containerd.ToAPIEndpoint(registryMirror.BaseRegistry) values["insecureSkip"] = registryMirror.InsecureSkipVerify values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror()) values["coreEKSAMirror"] = registryMirror.CoreEKSAMirror() diff --git a/pkg/providers/vsphere/config/template-cp.yaml b/pkg/providers/vsphere/config/template-cp.yaml index 8cd5eb3f19b1..9f89dc4e68ba 100644 --- a/pkg/providers/vsphere/config/template-cp.yaml +++ b/pkg/providers/vsphere/config/template-cp.yaml @@ -383,9 +383,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -395,7 +396,7 @@ spec: {{- end }} {{- end }} {{- if .registryAuth }} - [host."https://{{ .mirrorBase }}".header] + [host."https://{{ .mirrorBaseAPIEndpoint }}".header] authorization = "Basic {{ printf "%s:%s" .registryUsername .registryPassword | b64enc }}" {{- end }} owner: root:root @@ -403,9 +404,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/vsphere/config/template-md.yaml b/pkg/providers/vsphere/config/template-md.yaml index e1d89da073b1..b6f1242f1581 100644 --- a/pkg/providers/vsphere/config/template-md.yaml +++ b/pkg/providers/vsphere/config/template-md.yaml @@ -130,9 +130,10 @@ spec: path: "/etc/containerd/config_append.toml" - content: | server = "https://{{ .mirrorBase }}" - - [host."https://{{ .mirrorBase }}"] + + [host."https://{{ .mirrorBaseAPIEndpoint }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or .registryCACert .insecureSkip }} {{- if .registryCACert }} ca = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" @@ -142,7 +143,7 @@ spec: {{- end }} {{- end }} {{- if .registryAuth }} - [host."https://{{ .mirrorBase }}".header] + [host."https://{{ .mirrorBaseAPIEndpoint }}".header] authorization = "Basic {{ printf "%s:%s" .registryUsername .registryPassword | b64enc }}" {{- end }} owner: root:root @@ -150,9 +151,10 @@ spec: {{- range $orig, $mirror := .registryMirrorMap }} - content: | server = "https://{{ $orig }}" - + [host."https://{{ $mirror }}"] capabilities = ["pull", "resolve"] + override_path = true {{- if or $.registryCACert $.insecureSkip }} {{- if $.registryCACert }} ca = "/etc/containerd/certs.d/{{ $.mirrorBase }}/ca.crt" diff --git a/pkg/providers/vsphere/template.go b/pkg/providers/vsphere/template.go index 3929ccefca0c..0806a58007f1 100644 --- a/pkg/providers/vsphere/template.go +++ b/pkg/providers/vsphere/template.go @@ -260,6 +260,7 @@ func buildTemplateMapCP( registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) values["mirrorBase"] = registryMirror.BaseRegistry + values["mirrorBaseAPIEndpoint"] = containerd.ToAPIEndpoint(registryMirror.BaseRegistry) values["insecureSkip"] = registryMirror.InsecureSkipVerify values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror()) if len(registryMirror.CACertContent) > 0 { @@ -498,6 +499,7 @@ func buildTemplateMapMD( registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) values["mirrorBase"] = registryMirror.BaseRegistry + values["mirrorBaseAPIEndpoint"] = containerd.ToAPIEndpoint(registryMirror.BaseRegistry) values["insecureSkip"] = registryMirror.InsecureSkipVerify values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror()) if len(registryMirror.CACertContent) > 0 { diff --git a/pkg/registrymirror/containerd/utils.go b/pkg/registrymirror/containerd/utils.go index 9b09a2cf0ef5..d073cf6c54fd 100644 --- a/pkg/registrymirror/containerd/utils.go +++ b/pkg/registrymirror/containerd/utils.go @@ -16,6 +16,8 @@ func ToAPIEndpoint(url string) string { } if u.Path != "" { u.Path = filepath.Join("v2", u.Path) + } else { + u.Path = "v2" } return strings.TrimPrefix(u.String(), "//") } diff --git a/pkg/registrymirror/containerd/utils_test.go b/pkg/registrymirror/containerd/utils_test.go index a86030f7954a..ec77884421ac 100644 --- a/pkg/registrymirror/containerd/utils_test.go +++ b/pkg/registrymirror/containerd/utils_test.go @@ -18,12 +18,12 @@ func TestToAPIEndpoint(t *testing.T) { { name: "no namespace", URL: "oci://1.2.3.4:443", - want: "oci://1.2.3.4:443", + want: "oci://1.2.3.4:443/v2", }, { name: "no namespace", URL: "registry-mirror.test:443", - want: "registry-mirror.test:443", + want: "registry-mirror.test:443/v2", }, { name: "with namespace", @@ -57,7 +57,7 @@ func TestToAPIEndpoints(t *testing.T) { constants.DefaultCuratedPackagesRegistry: "1.2.3.4:443/curated-packages", }, want: map[string]string{ - constants.DefaultCoreEKSARegistry: "1.2.3.4:443", + constants.DefaultCoreEKSARegistry: "1.2.3.4:443/v2", constants.DefaultCuratedPackagesRegistry: "1.2.3.4:443/v2/curated-packages", }, },