diff --git a/pkg/ingress2gateway/migrate_test.go b/pkg/ingress2gateway/migrate_test.go new file mode 100644 index 000000000..b9dfc7b12 --- /dev/null +++ b/pkg/ingress2gateway/migrate_test.go @@ -0,0 +1,37 @@ +package ingress2gateway_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/aws-load-balancer-controller/pkg/ingress2gateway" + "sigs.k8s.io/aws-load-balancer-controller/pkg/ingress2gateway/reader" + "sigs.k8s.io/aws-load-balancer-controller/pkg/ingress2gateway/translate" + "sigs.k8s.io/aws-load-balancer-controller/pkg/ingress2gateway/writer" +) + +func TestMigrate_VanillaIngress(t *testing.T) { + inputFile := filepath.Join("testdata", "vanilla_ingress.yaml") + expectedFile := filepath.Join("testdata", "expected_vanilla_gateway.yaml") + + outputDir := t.TempDir() + + err := ingress2gateway.Migrate(context.Background(), ingress2gateway.MigrateOptions{ + Files: []string{inputFile}, + OutputDir: outputDir, + OutputFormat: "yaml", + }, reader.Read, translate.Translate, writer.Write) + require.NoError(t, err) + + actual, err := os.ReadFile(filepath.Join(outputDir, "gateway-resources.yaml")) + require.NoError(t, err) + + expected, err := os.ReadFile(expectedFile) + require.NoError(t, err) + + assert.Equal(t, string(expected), string(actual), "generated gateway manifest does not match expected") +} diff --git a/pkg/ingress2gateway/testdata/expected_vanilla_gateway.yaml b/pkg/ingress2gateway/testdata/expected_vanilla_gateway.yaml new file mode 100644 index 000000000..50bd95863 --- /dev/null +++ b/pkg/ingress2gateway/testdata/expected_vanilla_gateway.yaml @@ -0,0 +1,86 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: aws-alb +spec: + controllerName: gateway.k8s.aws/alb +--- +apiVersion: gateway.k8s.aws/v1beta1 +kind: LoadBalancerConfiguration +metadata: + name: vanilla-lb-confi-17d9f29244 + namespace: vanilla-ns +spec: + scheme: internet-facing + tags: + gateway.k8s.aws/migrated-from: ingress/vanilla-ns/vanilla-ingress +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: vanilla-gateway-86d474571f + namespace: vanilla-ns +spec: + gatewayClassName: aws-alb + infrastructure: + parametersRef: + group: gateway.k8s.aws + kind: LoadBalancerConfiguration + name: vanilla-lb-confi-17d9f29244 + listeners: + - name: http-80 + port: 80 + protocol: HTTP +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: vanilla-route-a166c439e9 + namespace: vanilla-ns +spec: + hostnames: + - app.example.com + parentRefs: + - name: vanilla-gateway-86d474571f + sectionName: http-80 + rules: + - backendRefs: + - name: svc-api + port: 80 + matches: + - path: + type: PathPrefix + value: /api + - backendRefs: + - name: svc-web + port: 80 + matches: + - path: + type: PathPrefix + value: /web +--- +apiVersion: gateway.k8s.aws/v1beta1 +kind: TargetGroupConfiguration +metadata: + name: svc-api-tg-confi-076740c021 + namespace: vanilla-ns +spec: + defaultConfiguration: + tags: + gateway.k8s.aws/migrated-from: ingress/vanilla-ns/vanilla-ingress + targetType: ip + targetReference: + name: svc-api +--- +apiVersion: gateway.k8s.aws/v1beta1 +kind: TargetGroupConfiguration +metadata: + name: svc-web-tg-confi-72203f1b04 + namespace: vanilla-ns +spec: + defaultConfiguration: + tags: + gateway.k8s.aws/migrated-from: ingress/vanilla-ns/vanilla-ingress + targetType: ip + targetReference: + name: svc-web diff --git a/pkg/ingress2gateway/testdata/vanilla_ingress.yaml b/pkg/ingress2gateway/testdata/vanilla_ingress.yaml new file mode 100644 index 000000000..aaf5c5765 --- /dev/null +++ b/pkg/ingress2gateway/testdata/vanilla_ingress.yaml @@ -0,0 +1,35 @@ +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: vanilla-class +spec: + controller: ingress.k8s.aws/alb +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + namespace: vanilla-ns + name: vanilla-ingress + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip +spec: + ingressClassName: vanilla-class + rules: + - host: app.example.com + http: + paths: + - path: /api + pathType: Prefix + backend: + service: + name: svc-api + port: + number: 80 + - path: /web + pathType: Prefix + backend: + service: + name: svc-web + port: + number: 80 diff --git a/pkg/ingress2gateway/translate/translate.go b/pkg/ingress2gateway/translate/translate.go index 719483541..e8afa7eed 100644 --- a/pkg/ingress2gateway/translate/translate.go +++ b/pkg/ingress2gateway/translate/translate.go @@ -3,6 +3,7 @@ package translate import ( "fmt" "os" + "sort" "strings" corev1 "k8s.io/api/core/v1" @@ -182,8 +183,14 @@ func Translate(in *ingress2gateway.InputResources) (*ingress2gateway.OutputResou } // Build TargetGroupConfigurations from accumulated entries. - for _, entries := range tgcEntries { - tgc := buildTargetGroupConfigFromEntries(entries) + // Sort keys for deterministic output order. + tgcKeys := make([]string, 0, len(tgcEntries)) + for k := range tgcEntries { + tgcKeys = append(tgcKeys, k) + } + sort.Strings(tgcKeys) + for _, key := range tgcKeys { + tgc := buildTargetGroupConfigFromEntries(tgcEntries[key]) if tgc != nil { out.TargetGroupConfigurations = append(out.TargetGroupConfigurations, *tgc) } diff --git a/test/e2e/ingress2gateway/builders.go b/test/e2e/ingress2gateway/builders.go new file mode 100644 index 000000000..28c4c0c0c --- /dev/null +++ b/test/e2e/ingress2gateway/builders.go @@ -0,0 +1,114 @@ +package ingress2gateway + +import ( + "maps" + + networking "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ingressGroupOutput holds the built resources ready for creation. +type ingressGroupOutput struct { + IngressClass *networking.IngressClass + Ingresses []*networking.Ingress +} + +// buildBasicIngressGroup builds an IngressGroup with 2 members: +// - member-1: admin.example.com / -> svcC +// - member-2: app.example.com /api -> svcA, /health -> svcB +// +// Includes health check annotations and user tags. +func buildBasicIngressGroup(namespace, groupName string, ipFamily string, svcAName, svcBName, svcCName string) ingressGroupOutput { + pathPrefix := networking.PathTypePrefix + ingressClassName := namespace + + ingClass := &networking.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: ingressClassName, + }, + Spec: networking.IngressClassSpec{ + Controller: ingressController, + }, + } + + sharedAnnotations := map[string]string{ + annotationScheme: "internet-facing", + annotationTargetType: "ip", + annotationGroupName: groupName, + annotationTags: "Team=e2e,Component=migration", + annotationHealthCheckPath: pathHealth, + annotationHealthyThreshold: "2", + annotationUnhealthyThreshold: "3", + annotationHealthCheckInterval: "15", + } + if ipFamily == "IPv6" { + sharedAnnotations[annotationIPAddressType] = "dualstack" + } + + ing1Annotations := maps.Clone(sharedAnnotations) + ing1Annotations[annotationGroupOrder] = "1" + + ing2Annotations := maps.Clone(sharedAnnotations) + ing2Annotations[annotationGroupOrder] = "2" + + ing1 := &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "member-1", + Annotations: ing1Annotations, + }, + Spec: networking.IngressSpec{ + IngressClassName: &ingressClassName, + Rules: []networking.IngressRule{ + { + Host: hostAdmin, + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + {Path: pathRoot, PathType: &pathPrefix, Backend: ingressBackend(svcCName)}, + }, + }, + }, + }, + }, + }, + } + + ing2 := &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "member-2", + Annotations: ing2Annotations, + }, + Spec: networking.IngressSpec{ + IngressClassName: &ingressClassName, + Rules: []networking.IngressRule{ + { + Host: hostApp, + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + {Path: pathAPI, PathType: &pathPrefix, Backend: ingressBackend(svcAName)}, + {Path: pathHealth, PathType: &pathPrefix, Backend: ingressBackend(svcBName)}, + }, + }, + }, + }, + }, + }, + } + + return ingressGroupOutput{ + IngressClass: ingClass, + Ingresses: []*networking.Ingress{ing1, ing2}, + } +} + +func ingressBackend(svcName string) networking.IngressBackend { + return networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svcName, + Port: networking.ServiceBackendPort{Number: 80}, + }, + } +} diff --git a/test/e2e/ingress2gateway/constants.go b/test/e2e/ingress2gateway/constants.go new file mode 100644 index 000000000..ab61448c7 --- /dev/null +++ b/test/e2e/ingress2gateway/constants.go @@ -0,0 +1,32 @@ +package ingress2gateway + +const ( + ingressController = "ingress.k8s.aws/alb" + + annotationScheme = "alb.ingress.kubernetes.io/scheme" + annotationTargetType = "alb.ingress.kubernetes.io/target-type" + annotationGroupName = "alb.ingress.kubernetes.io/group.name" + annotationGroupOrder = "alb.ingress.kubernetes.io/group.order" + annotationTags = "alb.ingress.kubernetes.io/tags" + annotationHealthCheckPath = "alb.ingress.kubernetes.io/healthcheck-path" + annotationHealthyThreshold = "alb.ingress.kubernetes.io/healthy-threshold-count" + annotationUnhealthyThreshold = "alb.ingress.kubernetes.io/unhealthy-threshold-count" + annotationHealthCheckInterval = "alb.ingress.kubernetes.io/healthcheck-interval-seconds" + annotationIPAddressType = "alb.ingress.kubernetes.io/ip-address-type" + annotationDryRunPlan = "alb.ingress.kubernetes.io/dry-run-plan" + + gwDryRunAnnotation = "gateway.k8s.aws/dry-run" + gwDryRunPlan = "gateway.k8s.aws/dry-run-plan" + migrationTagKey = "gateway.k8s.aws/migrated-from" + + hostAdmin = "admin.example.com" + hostApp = "app.example.com" + + pathRoot = "/" + pathAPI = "/api" + pathHealth = "/health" + + bodyServiceA = "service-a" + bodyServiceB = "service-b" + bodyServiceC = "service-c" +) diff --git a/test/e2e/ingress2gateway/ingress2gateway_suite_test.go b/test/e2e/ingress2gateway/ingress2gateway_suite_test.go new file mode 100644 index 000000000..635b821dc --- /dev/null +++ b/test/e2e/ingress2gateway/ingress2gateway_suite_test.go @@ -0,0 +1,34 @@ +package ingress2gateway + +import ( + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/aws-load-balancer-controller/test/framework" +) + +var tf *framework.Framework + +func TestIngress2Gateway(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Ingress2Gateway Suite") +} + +var _ = SynchronizedBeforeSuite(func() []byte { + var err error + tf, err = framework.InitFramework() + Expect(err).NotTo(HaveOccurred()) + + if tf.Options.ControllerImage != "" { + err = tf.CTRLInstallationManager.UpgradeController(tf.Options.ControllerImage, true, true, false) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(60 * time.Second) + } + return nil +}, func(data []byte) { + var err error + tf, err = framework.InitFramework() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/test/e2e/ingress2gateway/migration_test.go b/test/e2e/ingress2gateway/migration_test.go new file mode 100644 index 000000000..ef2bb52ce --- /dev/null +++ b/test/e2e/ingress2gateway/migration_test.go @@ -0,0 +1,205 @@ +package ingress2gateway + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/aws-load-balancer-controller/test/framework" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/manifest" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("ingress to gateway migration", func() { + var ( + ctx context.Context + sandboxNS *corev1.Namespace + ingClass *networking.IngressClass + ) + + BeforeEach(func() { + if !tf.Options.EnableMigrationTests { + Skip("Skipping migration tests") + } + ctx = context.Background() + By("setup sandbox namespace") + ns, err := tf.NSManager.AllocateNamespace(ctx, "i2g-e2e") + Expect(err).NotTo(HaveOccurred()) + sandboxNS = ns + }) + + AfterEach(func() { + if sandboxNS != nil { + By("teardown sandbox namespace") + ingList := &networking.IngressList{} + _ = tf.K8sClient.List(ctx, ingList, client.InNamespace(sandboxNS.Name)) + for i := range ingList.Items { + _ = tf.K8sClient.Delete(ctx, &ingList.Items[i]) + } + for i := range ingList.Items { + _ = tf.INGManager.WaitUntilIngressDeleted(ctx, &ingList.Items[i]) + } + deleteGatewayResources(ctx, tf, sandboxNS.Name) + if ingClass != nil { + _ = tf.K8sClient.Delete(ctx, ingClass) + } + err := tf.K8sClient.Delete(ctx, sandboxNS) + Expect(err).Should(SatisfyAny(BeNil(), Satisfy(apierrs.IsNotFound))) + err = tf.NSManager.WaitUntilNamespaceDeleted(ctx, sandboxNS) + Expect(err).NotTo(HaveOccurred()) + } + }) + + It("should migrate an IngressGroup with multiple paths, tags, and health checks through dry-run and full cutover", func() { + groupName := fmt.Sprintf("e2e-mig-%s", sandboxNS.Name) + + // --- Phase 1: Create Ingress baseline --- + By("creating backend services") + appBuilder := manifest.NewFixedResponseServiceBuilder() + + dpA, svcA := appBuilder.WithHTTPBody(bodyServiceA).Build(sandboxNS.Name, "svc-a", tf.Options.TestImageRegistry) + dpB, svcB := appBuilder.WithHTTPBody(bodyServiceB).Build(sandboxNS.Name, "svc-b", tf.Options.TestImageRegistry) + dpC, svcC := appBuilder.WithHTTPBody(bodyServiceC).Build(sandboxNS.Name, "svc-c", tf.Options.TestImageRegistry) + + for _, obj := range []client.Object{dpA, svcA, dpB, svcB, dpC, svcC} { + Expect(tf.K8sClient.Create(ctx, obj)).To(Succeed()) + } + _, err := tf.DPManager.WaitUntilDeploymentReady(ctx, dpA) + Expect(err).NotTo(HaveOccurred()) + _, err = tf.DPManager.WaitUntilDeploymentReady(ctx, dpB) + Expect(err).NotTo(HaveOccurred()) + _, err = tf.DPManager.WaitUntilDeploymentReady(ctx, dpC) + Expect(err).NotTo(HaveOccurred()) + + By("creating IngressGroup (2 members)") + ipFamily := "" + if tf.Options.IPFamily == framework.IPv6 { + ipFamily = "IPv6" + } + group := buildBasicIngressGroup(sandboxNS.Name, groupName, ipFamily, svcA.Name, svcB.Name, svcC.Name) + + ingClass = group.IngressClass + Expect(tf.K8sClient.Create(ctx, ingClass)).To(Succeed()) + + expectedTraffic := []trafficCase{ + {pathRoot, hostAdmin, bodyServiceC}, + {pathAPI, hostApp, bodyServiceA}, + {pathHealth, hostApp, bodyServiceB}, + } + + for _, ing := range group.Ingresses { + Expect(tf.K8sClient.Create(ctx, ing)).To(Succeed()) + } + + By("waiting for Ingress ALB to be provisioned") + primaryIng := group.Ingresses[0] + var ingressLBDNS string + Eventually(func(g Gomega) { + err := tf.K8sClient.Get(ctx, client.ObjectKeyFromObject(primaryIng), primaryIng) + g.Expect(err).NotTo(HaveOccurred()) + ingressLBDNS = findIngressDNS(primaryIng) + g.Expect(ingressLBDNS).NotTo(BeEmpty()) + }, utils.CertReconcileTimeout, utils.PollIntervalShort).Should(Succeed()) + tf.Logger.Info("ingress DNS populated", "dns", ingressLBDNS) + + ingressLBARN, err := tf.LBManager.FindLoadBalancerByDNSName(ctx, ingressLBDNS) + Expect(err).NotTo(HaveOccurred()) + Expect(ingressLBARN).NotTo(BeEmpty()) + + tf.Logger.Info("waiting for ingress ALB to be available") + err = tf.LBManager.WaitUntilLoadBalancerAvailable(ctx, ingressLBARN) + Expect(err).NotTo(HaveOccurred()) + + By("verifying traffic to ingress ALB", func() { + ExpectLBDNSResolvable(ctx, tf, ingressLBDNS) + verifyTraffic(tf, ingressLBDNS, expectedTraffic) + }) + + By("verifying dry-run-plan annotation on primary member only", func() { + expectDryRunPlan(ctx, tf, group.Ingresses[0], group.Ingresses[1:]...) + }) + + // --- Phase 2: Run migration tool --- + outputDir, err := os.MkdirTemp("", "i2g-e2e-*") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(outputDir) + + By("running lbc-migrate tool", func() { + err = runMigrateTool(sandboxNS.Name, outputDir, "--dry-run") + Expect(err).NotTo(HaveOccurred()) + }) + + By("verifying migration output files exist", func() { + outputFiles, err := filepath.Glob(filepath.Join(outputDir, "*.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(outputFiles).NotTo(BeEmpty()) + }) + + // --- Phase 3: Apply in dry-run mode and verify --- + By("applying generated Gateway manifests (dry-run mode)", func() { + err = applyYAMLDir(outputDir) + Expect(err).NotTo(HaveOccurred()) + }) + + By("verifying generated resource counts", func() { + verifyResourceCounts(ctx, tf, sandboxNS.Name, expectedResourceCounts{ + gateways: 1, + httpRoutes: 2, + loadBalancerConfigurations: 1, + targetGroupConfigurations: 3, + listenerRuleConfigurations: 3, + }) + }) + + gw := findGatewayInNamespace(ctx, tf, sandboxNS.Name) + Expect(gw).NotTo(BeNil(), "no Gateway found in namespace %s", sandboxNS.Name) + + By("verifying Gateway has dry-run annotation", func() { + Expect(gw.Annotations[gwDryRunAnnotation]).To(Equal("true")) + }) + + By("waiting for Gateway dry-run-plan annotation", func() { + expectGatewayDryRunPlan(ctx, tf, gw) + }) + + By("verifying NO ALB is created for dry-run Gateway", func() { + expectNoALBForGateway(ctx, tf, gw) + }) + + // --- Phase 4: Full cutover --- + By("removing dry-run annotation to trigger real reconciliation", func() { + removeDryRunAnnotation(ctx, tf, gw) + }) + + By("waiting for Gateway ALB to be provisioned") + gatewayLBARN, gatewayLBDNS := expectGatewayALBProvisioned(ctx, tf, gw) + + tf.Logger.Info("waiting for gateway ALB to be available") + err = tf.LBManager.WaitUntilLoadBalancerAvailable(ctx, gatewayLBARN) + Expect(err).NotTo(HaveOccurred()) + + By("comparing ALB configurations: listeners, rules, and target groups", func() { + compareALBConfigurations(ctx, tf, ingressLBARN, gatewayLBARN) + }) + + By("comparing ALB configurations: tags", func() { + migrationTagLBValue := fmt.Sprintf("ingress-group/%s", groupName) + compareTags(ctx, tf, ingressLBARN, gatewayLBARN, migrationTagLBValue, map[string]string{ + "Team": "e2e", + "Component": "migration", + }) + }) + + By("verifying traffic to Gateway ALB", func() { + ExpectLBDNSResolvable(ctx, tf, gatewayLBDNS) + verifyTraffic(tf, gatewayLBDNS, expectedTraffic) + }) + }) +}) diff --git a/test/e2e/ingress2gateway/utils.go b/test/e2e/ingress2gateway/utils.go new file mode 100644 index 000000000..fe34ae690 --- /dev/null +++ b/test/e2e/ingress2gateway/utils.go @@ -0,0 +1,151 @@ +package ingress2gateway + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + + . "github.com/onsi/gomega" + networking "k8s.io/api/networking/v1" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/test/framework" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// runMigrateTool runs lbc-migrate as a subprocess with the given extra flags. +// When namespace is non-empty, --from-cluster and --namespace are added automatically. +func runMigrateTool(namespace, outputDir string, extraArgs ...string) error { + args := []string{"--output-dir", outputDir} + if namespace != "" { + args = append(args, "--from-cluster", "--namespaces", namespace) + } + args = append(args, extraArgs...) + cmd := exec.Command("lbc-migrate", args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("lbc-migrate failed: %v\noutput: %s", err, string(out)) + } + return nil +} + +// expectDryRunPlan waits for the dry-run-plan annotation to appear on the primary ingress +// and verifies it is absent on all secondary members, using a single polling loop. +func expectDryRunPlan(ctx context.Context, tf *framework.Framework, primary *networking.Ingress, secondaries ...*networking.Ingress) string { + var plan string + Eventually(func(g Gomega) { + err := tf.K8sClient.Get(ctx, client.ObjectKeyFromObject(primary), primary) + g.Expect(err).NotTo(HaveOccurred()) + plan = primary.Annotations[annotationDryRunPlan] + g.Expect(plan).ShouldNot(BeEmpty()) + + for _, sec := range secondaries { + err = tf.K8sClient.Get(ctx, client.ObjectKeyFromObject(sec), sec) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(sec.Annotations[annotationDryRunPlan]).To(BeEmpty()) + } + }, utils.IngressReconcileTimeout, utils.PollIntervalShort).Should(Succeed()) + return plan +} + +// expectGatewayDryRunPlan waits for the gateway controller to write the dry-run-plan annotation. +func expectGatewayDryRunPlan(ctx context.Context, tf *framework.Framework, gw *gwv1.Gateway) string { + var plan string + Eventually(func(g Gomega) { + err := tf.K8sClient.Get(ctx, client.ObjectKeyFromObject(gw), gw) + g.Expect(err).NotTo(HaveOccurred()) + plan = gw.Annotations[gwDryRunPlan] + g.Expect(plan).ShouldNot(BeEmpty()) + }, utils.CertReconcileTimeout, utils.PollIntervalShort).Should(Succeed()) + return plan +} + +// expectNoALBForGateway verifies no ALB is created for a gateway (dry-run mode). +func expectNoALBForGateway(ctx context.Context, tf *framework.Framework, gw *gwv1.Gateway) { + Consistently(func(g Gomega) { + err := tf.K8sClient.Get(ctx, client.ObjectKeyFromObject(gw), gw) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(gw.Status.Addresses).To(BeEmpty()) + }, utils.IngressReconcileTimeout, utils.PollIntervalShort).Should(Succeed()) +} + +// expectGatewayALBProvisioned waits for the gateway to get an ALB address and returns its ARN and DNS. +func expectGatewayALBProvisioned(ctx context.Context, tf *framework.Framework, gw *gwv1.Gateway) (string, string) { + var lbDNS string + Eventually(func(g Gomega) { + err := tf.K8sClient.Get(ctx, client.ObjectKeyFromObject(gw), gw) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(gw.Status.Addresses).NotTo(BeEmpty()) + lbDNS = gw.Status.Addresses[0].Value + g.Expect(lbDNS).NotTo(BeEmpty()) + }, utils.CertReconcileTimeout, utils.PollIntervalShort).Should(Succeed()) + + lbARN, err := tf.LBManager.FindLoadBalancerByDNSName(ctx, lbDNS) + Expect(err).NotTo(HaveOccurred()) + Expect(lbARN).NotTo(BeEmpty()) + return lbARN, lbDNS +} + +// removeDryRunAnnotation patches the gateway to remove the dry-run annotation, triggering real reconciliation. +func removeDryRunAnnotation(ctx context.Context, tf *framework.Framework, gw *gwv1.Gateway) { + gwOld := gw.DeepCopy() + delete(gw.Annotations, gwDryRunAnnotation) + err := tf.K8sClient.Patch(ctx, gw, client.MergeFrom(gwOld)) + Expect(err).NotTo(HaveOccurred()) +} + +// ExpectLBDNSResolvable waits for the LB DNS to become resolvable. +func ExpectLBDNSResolvable(ctx context.Context, tf *framework.Framework, lbDNS string) { + tf.Logger.Info("waiting for LB DNS to be resolvable", "dns", lbDNS) + dnsCtx, cancel := context.WithTimeout(ctx, tf.Options.DNSTimeout) + defer cancel() + err := utils.WaitUntilDNSNameAvailable(dnsCtx, lbDNS) + Expect(err).NotTo(HaveOccurred()) +} + +// findGatewayInNamespace finds the first Gateway in the namespace. +func findGatewayInNamespace(ctx context.Context, tf *framework.Framework, namespace string) *gwv1.Gateway { + gwList := &gwv1.GatewayList{} + err := tf.K8sClient.List(ctx, gwList, client.InNamespace(namespace)) + Expect(err).NotTo(HaveOccurred()) + if len(gwList.Items) == 0 { + return nil + } + return &gwList.Items[0] +} + +// applyYAMLDir applies all YAML files in a directory using kubectl. +func applyYAMLDir(dir string) error { + files, err := filepath.Glob(filepath.Join(dir, "*.yaml")) + if err != nil { + return err + } + for _, f := range files { + out, err := exec.Command("kubectl", "apply", "-f", f).CombinedOutput() + if err != nil { + return fmt.Errorf("kubectl apply failed for %s: %v\noutput: %s", f, err, out) + } + } + return nil +} + +// deleteGatewayResources deletes all Gateway API and LBC CRD resources in a namespace. +func deleteGatewayResources(ctx context.Context, tf *framework.Framework, namespace string) { + ns := client.InNamespace(namespace) + _ = tf.K8sClient.DeleteAllOf(ctx, &elbv2gw.ListenerRuleConfiguration{}, ns) + _ = tf.K8sClient.DeleteAllOf(ctx, &elbv2gw.TargetGroupConfiguration{}, ns) + _ = tf.K8sClient.DeleteAllOf(ctx, &elbv2gw.LoadBalancerConfiguration{}, ns) + _ = tf.K8sClient.DeleteAllOf(ctx, &gwv1.HTTPRoute{}, ns) + _ = tf.K8sClient.DeleteAllOf(ctx, &gwv1.Gateway{}, ns) +} + +func findIngressDNS(ing *networking.Ingress) string { + for _, i := range ing.Status.LoadBalancer.Ingress { + if i.Hostname != "" { + return i.Hostname + } + } + return "" +} diff --git a/test/e2e/ingress2gateway/verifiers.go b/test/e2e/ingress2gateway/verifiers.go new file mode 100644 index 000000000..25440dd95 --- /dev/null +++ b/test/e2e/ingress2gateway/verifiers.go @@ -0,0 +1,308 @@ +package ingress2gateway + +import ( + "context" + "fmt" + "slices" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + . "github.com/onsi/gomega" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/test/framework" + frameworkhttp "sigs.k8s.io/aws-load-balancer-controller/test/framework/http" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// compareALBConfigurations fetches listeners once for both ALBs and compares listeners, rules, target groups. +func compareALBConfigurations(ctx context.Context, tf *framework.Framework, ingressLBARN, gatewayLBARN string) { + listeners1, err := tf.LBManager.GetLoadBalancerListeners(ctx, ingressLBARN) + Expect(err).NotTo(HaveOccurred()) + listeners2, err := tf.LBManager.GetLoadBalancerListeners(ctx, gatewayLBARN) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(listeners2)).To(Equal(len(listeners1)), "listener count mismatch") + Expect(listenerPortSet(listeners2)).To(Equal(listenerPortSet(listeners1)), "listener ports mismatch") + + listenersByPort1 := listenerByPort(listeners1) + listenersByPort2 := listenerByPort(listeners2) + + for port, l1 := range listenersByPort1 { + l2, ok := listenersByPort2[port] + Expect(ok).To(BeTrue(), "gateway missing listener on port %s", port) + + rules1, err := tf.LBManager.GetLoadBalancerListenerRules(ctx, awssdk.ToString(l1.ListenerArn)) + Expect(err).NotTo(HaveOccurred()) + rules2, err := tf.LBManager.GetLoadBalancerListenerRules(ctx, awssdk.ToString(l2.ListenerArn)) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(rules2)).To(Equal(len(rules1)), "listener rule count mismatch for port %s", port) + + nonDefault1 := filterNonDefaultRules(rules1) + nonDefault2 := filterNonDefaultRules(rules2) + for j := range nonDefault1 { + matchRuleConditions(nonDefault1[j], nonDefault2[j], j) + } + + default1 := findDefaultRule(rules1) + default2 := findDefaultRule(rules2) + Expect(default1).NotTo(BeNil(), "ingress listener on port %s missing default rule", port) + Expect(default2).NotTo(BeNil(), "gateway listener on port %s missing default rule", port) + matchDefaultRuleActions(default1, default2, port) + } + + tgs1, err := tf.TGManager.GetTargetGroupsForLoadBalancer(ctx, ingressLBARN) + Expect(err).NotTo(HaveOccurred()) + tgs2, err := tf.TGManager.GetTargetGroupsForLoadBalancer(ctx, gatewayLBARN) + Expect(err).NotTo(HaveOccurred()) + Expect(len(tgs2)).To(Equal(len(tgs1)), "target group count mismatch") + Expect(targetGroupTypeSet(tgs2)).To(Equal(targetGroupTypeSet(tgs1)), "target group types mismatch") +} + +// compareTags verifies tags on LB, listener rules, and target groups. +// Gateway resources should have all user tags from ingress plus the migrated-from tag. +func compareTags(ctx context.Context, tf *framework.Framework, ingressLBARN, gatewayLBARN, migrationTagLBValue string, expectedUserTags map[string]string) { + ingressLBTags := getTagMap(ctx, tf, ingressLBARN) + gatewayLBTags := getTagMap(ctx, tf, gatewayLBARN) + Expect(len(gatewayLBTags)).To(Equal(len(ingressLBTags)+1), "gateway LB should have exactly 1 more tag than ingress LB (migrated-from)") + for k, v := range expectedUserTags { + Expect(ingressLBTags).To(HaveKeyWithValue(k, v), "user tag %s=%s missing on ingress LB", k, v) + Expect(gatewayLBTags).To(HaveKeyWithValue(k, v), "user tag %s=%s missing on gateway LB", k, v) + } + Expect(gatewayLBTags).To(HaveKeyWithValue(migrationTagKey, migrationTagLBValue), + "migrated-from tag missing on gateway LB") + Expect(ingressLBTags).NotTo(HaveKey(migrationTagKey), + "migrated-from tag should not exist on ingress LB") + + ingressListeners, err := tf.LBManager.GetLoadBalancerListeners(ctx, ingressLBARN) + Expect(err).NotTo(HaveOccurred()) + gatewayListeners, err := tf.LBManager.GetLoadBalancerListeners(ctx, gatewayLBARN) + Expect(err).NotTo(HaveOccurred()) + + listenersByPort1 := listenerByPort(ingressListeners) + listenersByPort2 := listenerByPort(gatewayListeners) + + for port, l1 := range listenersByPort1 { + l2 := listenersByPort2[port] + ingressRules := getNonDefaultRules(ctx, tf, awssdk.ToString(l1.ListenerArn)) + gatewayRules := getNonDefaultRules(ctx, tf, awssdk.ToString(l2.ListenerArn)) + for j := range ingressRules { + ingressRuleTags := getTagMap(ctx, tf, awssdk.ToString(ingressRules[j].RuleArn)) + gatewayRuleTags := getTagMap(ctx, tf, awssdk.ToString(gatewayRules[j].RuleArn)) + Expect(len(gatewayRuleTags)).To(Equal(len(ingressRuleTags)+1), "gateway rule %d should have exactly 1 more tag than ingress rule (migrated-from)", j) + for k, v := range expectedUserTags { + Expect(ingressRuleTags).To(HaveKeyWithValue(k, v), "user tag %s=%s missing on ingress rule %d", k, v, j) + Expect(gatewayRuleTags).To(HaveKeyWithValue(k, v), "user tag %s=%s missing on gateway rule %d", k, v, j) + } + Expect(gatewayRuleTags).To(HaveKey(migrationTagKey), "migrated-from tag missing on gateway rule %d", j) + Expect(ingressRuleTags).NotTo(HaveKey(migrationTagKey), "migrated-from tag should not exist on ingress rule %d", j) + } + } + + ingressTGs, err := tf.TGManager.GetTargetGroupsForLoadBalancer(ctx, ingressLBARN) + Expect(err).NotTo(HaveOccurred()) + gatewayTGs, err := tf.TGManager.GetTargetGroupsForLoadBalancer(ctx, gatewayLBARN) + Expect(err).NotTo(HaveOccurred()) + + ingressTGsByHC := targetGroupByHealthCheck(ingressTGs) + gatewayTGsByHC := targetGroupByHealthCheck(gatewayTGs) + + for hc, ingTG := range ingressTGsByHC { + gwTG, ok := gatewayTGsByHC[hc] + Expect(ok).To(BeTrue(), "gateway missing TG with health check %s", hc) + + ingressTGTags := getTagMap(ctx, tf, awssdk.ToString(ingTG.TargetGroupArn)) + gatewayTGTags := getTagMap(ctx, tf, awssdk.ToString(gwTG.TargetGroupArn)) + Expect(len(gatewayTGTags)).To(Equal(len(ingressTGTags)+1), "gateway TG (hc=%s) should have exactly 1 more tag than ingress TG (migrated-from)", hc) + for k, v := range expectedUserTags { + Expect(ingressTGTags).To(HaveKeyWithValue(k, v), "user tag %s=%s missing on ingress TG (hc=%s)", k, v, hc) + Expect(gatewayTGTags).To(HaveKeyWithValue(k, v), "user tag %s=%s missing on gateway TG (hc=%s)", k, v, hc) + } + Expect(ingressTGTags).NotTo(HaveKey(migrationTagKey), "migrated-from tag should not exist on ingress TG") + Expect(gatewayTGTags).To(HaveKey(migrationTagKey), "migrated-from tag missing on gateway TG (hc=%s)", hc) + } +} + +// matchRuleConditions compares two rules by checking that they have the same number of conditions +// and that for each condition field in rule1, rule2 has a matching condition with the same values. +func matchRuleConditions(rule1, rule2 elbv2types.Rule, ruleIndex int) { + Expect(len(rule2.Conditions)).To(Equal(len(rule1.Conditions)), "rule %d condition count mismatch", ruleIndex) + + for _, c1 := range rule1.Conditions { + key := conditionKey(c1) + values1 := conditionValues(c1) + found := false + for _, c2 := range rule2.Conditions { + if conditionKey(c2) == key { + values2 := conditionValues(c2) + sorted1 := slices.Sorted(slices.Values(values1)) + sorted2 := slices.Sorted(slices.Values(values2)) + Expect(sorted2).To(Equal(sorted1), "rule %d condition %s values mismatch", ruleIndex, key) + found = true + break + } + } + Expect(found).To(BeTrue(), "rule %d condition %s not found in gateway rule", ruleIndex, key) + } +} + +func conditionKey(c elbv2types.RuleCondition) string { + field := awssdk.ToString(c.Field) + if c.HttpHeaderConfig != nil { + return fmt.Sprintf("%s/%s", field, awssdk.ToString(c.HttpHeaderConfig.HttpHeaderName)) + } + return field +} + +func conditionValues(c elbv2types.RuleCondition) []string { + if c.HostHeaderConfig != nil { + return c.HostHeaderConfig.Values + } + if c.PathPatternConfig != nil { + return c.PathPatternConfig.Values + } + if c.HttpHeaderConfig != nil { + return c.HttpHeaderConfig.Values + } + if c.HttpRequestMethodConfig != nil { + return c.HttpRequestMethodConfig.Values + } + if c.SourceIpConfig != nil { + return c.SourceIpConfig.Values + } + if c.QueryStringConfig != nil { + var values []string + for _, kv := range c.QueryStringConfig.Values { + values = append(values, fmt.Sprintf("%s=%s", awssdk.ToString(kv.Key), awssdk.ToString(kv.Value))) + } + return values + } + return nil +} + +func getTagMap(ctx context.Context, tf *framework.Framework, resourceARN string) map[string]string { + tags, err := tf.LBManager.GetLoadBalancerResourceTags(ctx, resourceARN) + Expect(err).NotTo(HaveOccurred()) + tagMap := make(map[string]string, len(tags)) + for _, t := range tags { + tagMap[awssdk.ToString(t.Key)] = awssdk.ToString(t.Value) + } + return tagMap +} + +func getNonDefaultRules(ctx context.Context, tf *framework.Framework, listenerARN string) []elbv2types.Rule { + rules, err := tf.LBManager.GetLoadBalancerListenerRules(ctx, listenerARN) + Expect(err).NotTo(HaveOccurred()) + return filterNonDefaultRules(rules) +} + +func listenerPortSet(listeners []elbv2types.Listener) map[string]string { + result := make(map[string]string) + for _, l := range listeners { + port := fmt.Sprintf("%d", awssdk.ToInt32(l.Port)) + result[port] = string(l.Protocol) + } + return result +} + +func listenerByPort(listeners []elbv2types.Listener) map[string]elbv2types.Listener { + result := make(map[string]elbv2types.Listener, len(listeners)) + for _, l := range listeners { + port := fmt.Sprintf("%d", awssdk.ToInt32(l.Port)) + result[port] = l + } + return result +} + +func targetGroupTypeSet(tgs []elbv2types.TargetGroup) map[string]int { + result := make(map[string]int) + for _, tg := range tgs { + result[string(tg.TargetType)]++ + } + return result +} + +func targetGroupByHealthCheck(tgs []elbv2types.TargetGroup) map[string]elbv2types.TargetGroup { + result := make(map[string]elbv2types.TargetGroup, len(tgs)) + for _, tg := range tgs { + key := fmt.Sprintf("%s:%s", awssdk.ToString(tg.HealthCheckPath), awssdk.ToString(tg.HealthCheckPort)) + result[key] = tg + } + return result +} + +func filterNonDefaultRules(rules []elbv2types.Rule) []elbv2types.Rule { + var result []elbv2types.Rule + for _, r := range rules { + if !awssdk.ToBool(r.IsDefault) { + result = append(result, r) + } + } + return result +} + +func findDefaultRule(rules []elbv2types.Rule) *elbv2types.Rule { + for i := range rules { + if awssdk.ToBool(rules[i].IsDefault) { + return &rules[i] + } + } + return nil +} + +func matchDefaultRuleActions(rule1, rule2 *elbv2types.Rule, port string) { + Expect(len(rule2.Actions)).To(Equal(len(rule1.Actions)), "default rule action count mismatch for port %s", port) + for i := range rule1.Actions { + Expect(string(rule2.Actions[i].Type)).To(Equal(string(rule1.Actions[i].Type)), + "default rule action %d type mismatch for port %s", i, port) + } +} + +type trafficCase struct { + path string + host string + expected string +} + +// verifyTraffic sends HTTP requests to the LB and verifies expected responses. +func verifyTraffic(tf *framework.Framework, lbDNS string, cases []trafficCase) { + for _, tc := range cases { + url := fmt.Sprintf("http://%s%s", lbDNS, tc.path) + err := tf.HTTPVerifier.VerifyURLWithOptions(url, frameworkhttp.URLOptions{ + HostHeader: tc.host, + }, frameworkhttp.ResponseCodeMatches(200), frameworkhttp.ResponseBodyMatches([]byte(tc.expected))) + Expect(err).NotTo(HaveOccurred(), "traffic verification failed for %s %s", tc.host, tc.path) + } +} + +type expectedResourceCounts struct { + gateways int + httpRoutes int + loadBalancerConfigurations int + targetGroupConfigurations int + listenerRuleConfigurations int +} + +// verifyResourceCounts checks that the migration produced exactly the expected number of each resource kind. +func verifyResourceCounts(ctx context.Context, tf *framework.Framework, namespace string, expected expectedResourceCounts) { + gwList := &gwv1.GatewayList{} + Expect(tf.K8sClient.List(ctx, gwList, client.InNamespace(namespace))).To(Succeed()) + Expect(gwList.Items).To(HaveLen(expected.gateways), "Gateway count mismatch") + + routeList := &gwv1.HTTPRouteList{} + Expect(tf.K8sClient.List(ctx, routeList, client.InNamespace(namespace))).To(Succeed()) + Expect(routeList.Items).To(HaveLen(expected.httpRoutes), "HTTPRoute count mismatch") + + lbcList := &elbv2gw.LoadBalancerConfigurationList{} + Expect(tf.K8sClient.List(ctx, lbcList, client.InNamespace(namespace))).To(Succeed()) + Expect(lbcList.Items).To(HaveLen(expected.loadBalancerConfigurations), "LoadBalancerConfiguration count mismatch") + + tgcList := &elbv2gw.TargetGroupConfigurationList{} + Expect(tf.K8sClient.List(ctx, tgcList, client.InNamespace(namespace))).To(Succeed()) + Expect(tgcList.Items).To(HaveLen(expected.targetGroupConfigurations), "TargetGroupConfiguration count mismatch") + + lrcList := &elbv2gw.ListenerRuleConfigurationList{} + Expect(tf.K8sClient.List(ctx, lrcList, client.InNamespace(namespace))).To(Succeed()) + Expect(lrcList.Items).To(HaveLen(expected.listenerRuleConfigurations), "ListenerRuleConfiguration count mismatch") +} diff --git a/test/framework/options.go b/test/framework/options.go index e1fa0d5cd..392166f7a 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -34,13 +34,14 @@ type Options struct { ControllerImage string // Additional parameters for e2e tests - S3BucketName string - CertificateARNs string - IPFamily string - TestImageRegistry string - EnableGatewayTests bool - EnableAGATests bool - EnableCertMgmtTests bool + S3BucketName string + CertificateARNs string + IPFamily string + TestImageRegistry string + EnableGatewayTests bool + EnableAGATests bool + EnableCertMgmtTests bool + EnableMigrationTests bool // ACM Certificate Management configuration for e2e test Route53ValidationDomain string @@ -74,6 +75,7 @@ func (options *Options) BindFlags() { flag.BoolVar(&options.EnableGatewayTests, "enable-gateway-tests", false, "enables gateway tests") flag.BoolVar(&options.EnableAGATests, "enable-aga-tests", false, "enables AWS Global Accelerator tests") flag.BoolVar(&options.EnableCertMgmtTests, "enable-cert-tests", false, "enables AWS ACM Certificate Management tests") + flag.BoolVar(&options.EnableMigrationTests, "enable-migration-tests", false, "enables ingress-to-gateway migration tests") flag.StringVar(&options.Route53ValidationDomain, "route53-validation-domain", "", `Route53 domain that can be used for requesting amazon_issued certificates`) flag.StringVar(&options.PCAARN, "pca-arn", "", `PCA ARN of CA that can be used to request private certificates`)