diff --git a/test/extended/router/config_manager_ingress.go b/test/extended/router/config_manager_ingress.go index 5325c72943c3..5c041c322df6 100644 --- a/test/extended/router/config_manager_ingress.go +++ b/test/extended/router/config_manager_ingress.go @@ -53,6 +53,7 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. ctx := context.Background() oc := exutil.NewCLIWithPodSecurityLevel("router-dcm-ingress", api.LevelPrivileged).AsAdmin() kubeClient := oc.AdminKubeClient() + routeClient := oc.AdminRouteClient() // variables updated on every new test var ( @@ -62,6 +63,20 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. ) g.AfterEach(func() { + if g.CurrentSpecReport().Failed() { + routes, _ := routeClient.RouteV1().Routes(oc.Namespace()).List(ctx, metav1.ListOptions{}) + if routes != nil { + outputIngress(routes.Items...) + } + endpoints, _ := kubeClient.CoreV1().Endpoints(oc.Namespace()).List(ctx, metav1.ListOptions{}) + if endpoints != nil { + outputEndpoints(endpoints.Items...) + } + epsList, _ := kubeClient.DiscoveryV1().EndpointSlices(oc.Namespace()).List(ctx, metav1.ListOptions{}) + if epsList != nil { + outputEndpointSlice(epsList.Items...) + } + } if controller.Name != "" { err := oc.AdminOperatorClient().OperatorV1().IngressControllers(controller.Namespace).Delete(ctx, controller.Name, *metav1.NewDeleteOptions(1)) o.Expect(err).NotTo(o.HaveOccurred()) @@ -73,7 +88,7 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. nsOperator := "openshift-ingress-operator" controllerName := names.SimpleNameGenerator.GenerateName("e2e-dcm-") - // ... and its router is created on router's namespace + // ... and its router and service are created in router's namespace nsRouter := "openshift-ingress" svcName := "router-internal-" + controllerName @@ -471,6 +486,7 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. // ... k8s recreates it and we wait it to be fully functional err = builder.waitDeployment(replicas, dcmIngressTimeout) + builder.printDeploymentState(ctx) o.Expect(err).NotTo(o.HaveOccurred()) } @@ -778,7 +794,12 @@ func (r *routeStackBuilder) createDeploymentStack(ctx context.Context, routetype if err = r.waitDeployment(replicas, timeout); err != nil { return nil, err } - return r.exposeDeployment(ctx) + backendServers, err = r.exposeDeployment(ctx) + if err != nil { + return nil, err + } + r.printDeploymentState(ctx) + return backendServers, nil } // scaleDeployment scales-in/out the common deployment to the specified replicas. It waits for all the pods to be created and returns their names. @@ -796,6 +817,7 @@ func (r *routeStackBuilder) scaleDeployment(ctx context.Context, replicas int, t } return len(backendServers) == replicas, nil }) + r.printDeploymentState(ctx) return backendServers, err } @@ -824,7 +846,14 @@ func (r *routeStackBuilder) createDetachedService(ctx context.Context) (serviceN } // we also need the deprecated Endpoints API, since router still uses it depending on the ROUTER_WATCH_ENDPOINTS envvar - epCurrent, err := r.kubeClient.CoreV1().Endpoints(svcCurrent.Namespace).Get(ctx, svcCurrent.Name, metav1.GetOptions{}) + var epCurrent *corev1.Endpoints + err = wait.PollUntilContextTimeout(ctx, time.Second, 5*time.Second, false, func(ctx context.Context) (done bool, err error) { + epCurrent, err = r.kubeClient.CoreV1().Endpoints(svcCurrent.Namespace).Get(ctx, svcCurrent.Name, metav1.GetOptions{}) + if err != nil { + framework.Logf("error fetching Endpoints: %s", err.Error()) + } + return err == nil, nil + }) if err != nil { return "", err } @@ -910,7 +939,7 @@ func (r *routeStackBuilder) scaleInEndpoints(ctx context.Context, detachedServic if err != nil { return err } - // deleting addresses, from all subnets, whose IP address is not found in the patched `eps` + // deleting addresses, from all subsets, whose IP address is not found in the patched `eps` for i := range ep.Subsets { ss := &ep.Subsets[i] ss.Addresses = slices.DeleteFunc(ss.Addresses, func(addr corev1.EndpointAddress) bool { @@ -953,6 +982,25 @@ func (r *routeStackBuilder) exposeDeployment(ctx context.Context) (backendServer return r.fetchServiceReplicas(ctx) } +// printDeploymentState outputs the pod names, status, and their IP addresses. Best effort, it outputs the error instead in case it happens. +// It requires that `exposeDeployment()` was already called. +func (r *routeStackBuilder) printDeploymentState(ctx context.Context) { + pods, err := r.fetchPods(ctx) + if err != nil { + framework.Logf("deployment state: error reading deployment pods: %v", err) + return + } + var podDescription []string + for _, pod := range pods { + var podIPs []string + for _, ip := range pod.Status.PodIPs { + podIPs = append(podIPs, ip.IP) + } + podDescription = append(podDescription, fmt.Sprintf("%s/%s/%s", pod.Name, pod.Status.Phase, strings.Join(podIPs, ","))) + } + framework.Logf("deployment state: replicas=%d pods=%s", len(pods), strings.Join(podDescription, " // ")) +} + // fetchEndpointSlice fetches the EndpointSlice of the provided service name. It currently supports only one EndpointSlice instance for simplicity. func (r *routeStackBuilder) fetchEndpointSlice(ctx context.Context, serviceName string) (*discoveryv1.EndpointSlice, error) { listOpts := metav1.ListOptions{LabelSelector: discoveryv1.LabelServiceName + "=" + serviceName} @@ -967,8 +1015,8 @@ func (r *routeStackBuilder) fetchEndpointSlice(ctx context.Context, serviceName return &epsList.Items[0], nil } -// fetchServiceReplicas fetches the pod names from the exposed common deployment. It requires that `exposeDeployment()` was already called. -func (r *routeStackBuilder) fetchServiceReplicas(ctx context.Context) ([]string, error) { +// fetchPods fetches the pods from the exposed common deployment. It requires that `exposeDeployment()` was already called. +func (r *routeStackBuilder) fetchPods(ctx context.Context) ([]corev1.Pod, error) { svc, err := r.kubeClient.CoreV1().Services(r.namespace).Get(ctx, r.resourceName, metav1.GetOptions{}) if err != nil { return nil, err @@ -978,9 +1026,18 @@ func (r *routeStackBuilder) fetchServiceReplicas(ctx context.Context) ([]string, if err != nil { return nil, err } - backendServers := make([]string, len(pods.Items)) - for i := range pods.Items { - backendServers[i] = pods.Items[i].Name + return pods.Items, nil +} + +// fetchServiceReplicas fetches the pod names from the exposed common deployment. It requires that `exposeDeployment()` was already called. +func (r *routeStackBuilder) fetchServiceReplicas(ctx context.Context) ([]string, error) { + pods, err := r.fetchPods(ctx) + if err != nil { + return nil, err + } + backendServers := make([]string, len(pods)) + for i := range pods { + backendServers[i] = pods[i].Name } return backendServers, nil } diff --git a/test/extended/router/stress.go b/test/extended/router/stress.go index 04c7699f8579..9d4c88f56df9 100644 --- a/test/extended/router/stress.go +++ b/test/extended/router/stress.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "strconv" "strings" "text/tabwriter" "time" @@ -16,6 +17,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -656,6 +658,64 @@ func outputIngress(routes ...routev1.Route) { e2e.Logf("Routes:\n%s", b.String()) } +func outputEndpoints(endpoints ...corev1.Endpoints) { + b := &bytes.Buffer{} + w := tabwriter.NewWriter(b, 0, 0, 2, ' ', 0) + fmt.Fprintf(w, "NAME\tADDRESSES\tNOT READY ADDRESSES\tPORTS\n") + for _, ep := range endpoints { + for _, ss := range ep.Subsets { + resumeAddrs := func(addrs []corev1.EndpointAddress) string { + var addrList []string + for _, addr := range addrs { + val := "-" + if addr.IP != "" { + val = addr.IP + } else if addr.Hostname != "" { + val = addr.Hostname + } + addrList = append(addrList, val) + } + return strings.Join(addrList, ",") + } + var portList []string + for _, port := range ss.Ports { + portList = append(portList, strconv.Itoa(int(port.Port))) + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", ep.Name, resumeAddrs(ss.Addresses), resumeAddrs(ss.NotReadyAddresses), strings.Join(portList, ",")) + } + } + w.Flush() + e2e.Logf("Endpoints:\n%s", b.String()) +} + +func outputEndpointSlice(epss ...discoveryv1.EndpointSlice) { + b := &bytes.Buffer{} + w := tabwriter.NewWriter(b, 0, 0, 2, ' ', 0) + fmt.Fprintf(w, "NAME\tSERVICE\tADDRESSES\tNOT READY ADDRESSES\tPORTS\n") + for _, eps := range epss { + var addrList, notReadyAddrList []string + for _, ep := range eps.Endpoints { + addrs := strings.Join(ep.Addresses, "+") + if ready := ep.Conditions.Ready; ready == nil || *ready == true { + addrList = append(addrList, addrs) + } else { + notReadyAddrList = append(notReadyAddrList, addrs) + } + } + var portList []string + for _, port := range eps.Ports { + val := "-" + if port.Port != nil { + val = strconv.Itoa(int(*port.Port)) + } + portList = append(portList, val) + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", eps.Name, eps.Labels[discoveryv1.LabelServiceName], strings.Join(addrList, ","), strings.Join(notReadyAddrList, ","), strings.Join(portList, ",")) + } + w.Flush() + e2e.Logf("EndpointSlices:\n%s", b.String()) +} + // findMostRecentConditionTime returns the time of the most recent condition. func findMostRecentConditionTime(conditions []routev1.RouteIngressCondition) time.Time { var recent time.Time