diff --git a/test/e2e/upgrade/upgrade_test.go b/test/e2e/upgrade/upgrade_test.go new file mode 100644 index 000000000..2238263e9 --- /dev/null +++ b/test/e2e/upgrade/upgrade_test.go @@ -0,0 +1,268 @@ +//go:build e2e + +// Package upgrade_test exercises a minimal Helm-chart upgrade path: +// 1. Install a previous release of the http-add-on chart. +// 2. Create a sample workload + HTTPScaledObject and verify it scales. +// 3. `helm upgrade` to the current chart version. +// 4. Verify the HTTPScaledObject still reconciles and the workload still scales. +// +// Skipped unless both HTTPADDON_UPGRADE_FROM_VERSION and HTTPADDON_UPGRADE_TO_VERSION +// are set (e.g. "0.10.0" and "0.11.0"). Intended to run as a dedicated CI job so the +// main e2e suite — which expects a single pre-installed add-on — is unaffected. +package upgrade_test + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/http-add-on/tests/helper" +) + +const ( + testName = "upgrade-httpso-test" + // releaseName and chart ref match the chart documented at + // https://github.com/kedacore/charts and referenced by tests/utils/setup_test.go. + releaseName = "http-add-on" + chartRef = "kedacore/http-add-on" + + fromVersionEnv = "HTTPADDON_UPGRADE_FROM_VERSION" + toVersionEnv = "HTTPADDON_UPGRADE_TO_VERSION" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + serviceName = fmt.Sprintf("%s-service", testName) + httpScaledObjectName = fmt.Sprintf("%s-http-so", testName) + host = testName + minReplicaCount = 0 + maxReplicaCount = 1 +) + +type templateData struct { + TestNamespace string + DeploymentName string + ServiceName string + HTTPScaledObjectName string + Host string + MinReplicas int + MaxReplicas int +} + +const ( + serviceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: {{.ServiceName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + ports: + - port: 8080 + targetPort: http + protocol: TCP + name: http + selector: + app: {{.DeploymentName}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}} + image: registry.k8s.io/e2e-test-images/agnhost:2.45 + args: + - netexec + ports: + - name: http + containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: / + port: http +` + + loadJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-request + namespace: {{.TestNamespace}} +spec: + template: + spec: + containers: + - name: curl-client + image: curlimages/curl + imagePullPolicy: Always + command: ["curl", "-H", "Host: {{.Host}}", "keda-add-ons-http-interceptor-proxy.keda:8080"] + restartPolicy: Never + activeDeadlineSeconds: 600 + backoffLimit: 5 +` + + httpScaledObjectTemplate = ` +kind: HTTPScaledObject +apiVersion: http.keda.sh/v1alpha1 +metadata: + name: {{.HTTPScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + hosts: + - {{.Host}} + targetPendingRequests: 100 + scaledownPeriod: 10 + scaleTargetRef: + name: {{.DeploymentName}} + service: {{.ServiceName}} + port: 8080 + replicas: + min: {{ .MinReplicas }} + max: {{ .MaxReplicas }} +` +) + +func TestUpgrade(t *testing.T) { + fromVersion := os.Getenv(fromVersionEnv) + toVersion := os.Getenv(toVersionEnv) + if fromVersion == "" || toVersion == "" { + t.Skipf("skipping upgrade scenario: set %s and %s to run", fromVersionEnv, toVersionEnv) + } + + kc := GetKubernetesClient(t) + + // The main runner (tests/run-all.go → tests/utils/setup_test.go) installs the + // add-on via `make deploy-e2e` (kustomize/ko), which creates the same + // keda-add-ons-http-* resources Helm wants to own. Tear those down first so + // the Helm release can take ownership cleanly. `make undeploy` is idempotent + // (it swallows errors if nothing is deployed) so this also handles the case + // where the test runs on a fresh cluster. + t.Log("--- tearing down any pre-existing add-on install ---") + undeployPreExistingAddon(t) + t.Cleanup(func() { + // Leave the cluster without the Helm release when we're done; a follow-up + // `make deploy-e2e` can reinstall via kustomize/ko. + _, _ = ExecuteCommand(fmt.Sprintf("helm uninstall %s --namespace %s --wait", releaseName, KEDANamespace)) + }) + + t.Logf("--- installing http-add-on %s (baseline) ---", fromVersion) + installOrUpgradeAddon(t, fromVersion) + waitForAddonReady(t, kc) + + data, templates := getTemplateData() + + t.Log("--- creating workload on baseline ---") + CreateKubernetesResources(t, kc, testNamespace, data, templates) + t.Cleanup(func() { + DeleteKubernetesResources(t, testNamespace, data, templates) + }) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 6, 10), + "replica count should be %d after setup", minReplicaCount) + + t.Log("--- scale out (baseline) ---") + testScaleOut(t, kc, data) + + t.Log("--- scale in (baseline) ---") + testScaleIn(t, kc, data) + + t.Logf("--- upgrading http-add-on to %s ---", toVersion) + installOrUpgradeAddon(t, toVersion) + waitForAddonReady(t, kc) + + // The upgrade itself takes longer than scaledownPeriod (10s), so any traffic + // applied before the upgrade will have drained by the time it finishes. + // Generate fresh load post-upgrade to prove the surviving HTTPScaledObject is + // still reconciled correctly and the interceptor still routes into it. + t.Log("--- scale out (post-upgrade) ---") + testScaleOut(t, kc, data) + + t.Log("--- scale in (post-upgrade) ---") + testScaleIn(t, kc, data) +} + +func undeployPreExistingAddon(t *testing.T) { + t.Helper() + // Runs from test/e2e/upgrade/ — repo root is three levels up. + out, err := ExecuteCommandWithDir("make undeploy", "../../..") + if err != nil { + // Not fatal — `make undeploy` may return non-zero on a clean cluster. + t.Logf("make undeploy returned error (may be harmless on a clean cluster): %s\n%s", err, string(out)) + } +} + +func installOrUpgradeAddon(t *testing.T, version string) { + t.Helper() + _, err := ExecuteCommand("helm repo add kedacore https://kedacore.github.io/charts") + require.NoErrorf(t, err, "cannot add kedacore helm repo - %s", err) + _, err = ExecuteCommand("helm repo update kedacore") + require.NoErrorf(t, err, "cannot update kedacore helm repo - %s", err) + _, err = ExecuteCommand(fmt.Sprintf( + "helm upgrade --install %s %s --version %s --namespace %s --wait", + releaseName, chartRef, version, KEDANamespace, + )) + require.NoErrorf(t, err, "cannot install/upgrade %s to version %s - %s", releaseName, version, err) +} + +func waitForAddonReady(t *testing.T, kc *kubernetes.Clientset) { + t.Helper() + // Same deployments TestSetupKEDA waits for — names are stable across the releases we support upgrading between. + for _, name := range []string{"keda-add-ons-http-operator", "keda-add-ons-http-interceptor", "keda-add-ons-http-external-scaler"} { + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, name, KEDANamespace, 1, 30, 6), + "%s not ready after upgrade", name) + } +} + +func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Helper() + KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate) + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 6, 10), + "replica count should be %d after load", maxReplicaCount) +} + +func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Helper() + KubectlDeleteWithTemplate(t, data, "loadJobTemplate", loadJobTemplate) + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 12, 10), + "replica count should be %d after load stops", minReplicaCount) +} + +func getTemplateData() (templateData, []Template) { + return templateData{ + TestNamespace: testNamespace, + DeploymentName: deploymentName, + ServiceName: serviceName, + HTTPScaledObjectName: httpScaledObjectName, + Host: host, + MinReplicas: minReplicaCount, + MaxReplicas: maxReplicaCount, + }, []Template{ + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "serviceNameTemplate", Config: serviceTemplate}, + {Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate}, + } +}