[RayService][Kueue] Support top-level Spec.Suspend

.buildkite/test-e2e.yml

@@ -55,9 +55,29 @@
     - set -o pipefail
     - mkdir -p "$(pwd)/tmp" && export KUBERAY_TEST_OUTPUT_DIR=$(pwd)/tmp
     - echo "KUBERAY_TEST_OUTPUT_DIR=$$KUBERAY_TEST_OUTPUT_DIR"
-    - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v ./test/e2erayservice 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-operator.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-rayservice-log.tar -T - && exit 1)
+    - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v -skip Suspend ./test/e2erayservice 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-operator.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-rayservice-log.tar -T - && exit 1)
     - echo "--- END:e2e rayservice (nightly operator) tests finished"
 
+- label: 'Test E2E rayservice suspend (nightly operator)'
+  instance_size: large
+  image: golang:1.26-bookworm
+  commands:
+    - source .buildkite/setup-env.sh
+    - kind create cluster --wait 900s --config ./ci/kind-config-buildkite.yml
+    - kubectl config set clusters.kind-kind.server https://docker:6443
+    # Build nightly KubeRay operator image
+    - pushd ray-operator
+    - source ../.buildkite/build-start-operator.sh
+    - kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator
+    # Run suspend e2e tests and print KubeRay operator logs if tests fail
+    - echo "--- START:Running e2e rayservice suspend (nightly operator) tests"
+    - if [ -n "$${KUBERAY_TEST_RAY_IMAGE}" ]; then echo "Using Ray Image $${KUBERAY_TEST_RAY_IMAGE}"; fi
+    - set -o pipefail
+    - mkdir -p "$(pwd)/tmp" && export KUBERAY_TEST_OUTPUT_DIR=$(pwd)/tmp
+    - echo "KUBERAY_TEST_OUTPUT_DIR=$$KUBERAY_TEST_OUTPUT_DIR"
+    - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v -run Suspend ./test/e2erayservice 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-operator.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-rayservice-suspend-log.tar -T - && exit 1)
+    - echo "--- END:e2e rayservice suspend (nightly operator) tests finished"
+
 - label: 'Test RayService Incremental Upgrade E2E (nightly operator)'
   instance_size: large
   image: golang:1.26-bookworm

docs/reference/api.md

@@ -505,6 +505,7 @@ _Appears in:_
 | `serveConfigV2` _string_ | Important: Run "make" to regenerate code after modifying this file<br />Defines the applications and deployments to deploy, should be a YAML multi-line scalar string. |  |  |
 | `rayClusterConfig` _[RayClusterSpec](#rayclusterspec)_ |  |  |  |
 | `excludeHeadPodFromServeSvc` _boolean_ | If the field is set to true, the value of the label `ray.io/serve` on the head Pod should always be false.<br />Therefore, the head Pod's endpoint will not be added to the Kubernetes Serve service. |  |  |
+| `suspend` _boolean_ | Suspend indicates whether the RayService should suspend its execution. When set to true,<br />all Kubernetes resources owned by the RayService controller (RayClusters, Kubernetes<br />Services, Gateway, HTTPRoute) will be deleted. Setting it back to false will allow the<br />RayService controller to recreate the resources. |  |  |
 
 
 

ray-operator/Makefile

@@ -81,8 +81,12 @@ test-e2e-autoscaler: manifests fmt vet ## Run e2e autoscaler tests.
 	go test -timeout 30m -v $(WHAT)
 
 test-e2e-rayservice: WHAT ?= ./test/e2erayservice
-test-e2e-rayservice: manifests fmt vet ## Run e2e RayService tests.
-	go test -timeout 30m -v $(WHAT)
+test-e2e-rayservice: manifests fmt vet ## Run e2e RayService tests (excluding suspend tests, which have their own target).
+	go test -timeout 30m -v -skip Suspend $(WHAT)
+
+test-e2e-rayservice-suspend: WHAT ?= ./test/e2erayservice
+test-e2e-rayservice-suspend: manifests fmt vet ## Run e2e RayService suspend tests.
+	go test -timeout 30m -v -run Suspend $(WHAT)
 
 test-e2e-upgrade: WHAT ?= ./test/e2eupgrade
 test-e2e-upgrade: manifests fmt vet ## Run e2e operator upgrade tests.

ray-operator/apis/ray/v1/rayservice_types.go

@@ -122,6 +122,12 @@ type RayServiceSpec struct {
 	// Therefore, the head Pod's endpoint will not be added to the Kubernetes Serve service.
 	// +optional
 	ExcludeHeadPodFromServeSvc bool `json:"excludeHeadPodFromServeSvc,omitempty"`
+	// Suspend indicates whether the RayService should suspend its execution. When set to true,
+	// all Kubernetes resources owned by the RayService controller (RayClusters, Kubernetes
+	// Services, Gateway, HTTPRoute) will be deleted. Setting it back to false will allow the
+	// RayService controller to recreate the resources.
+	// +optional
+	Suspend bool `json:"suspend,omitempty"`
 }
 
 // RayServiceStatuses defines the observed state of RayService
@@ -209,6 +215,11 @@ const (
 	UpgradeInProgress RayServiceConditionType = "UpgradeInProgress"
 	// RollbackInProgress means the RayService is currently rolling back an in-progress upgrade to the original cluster state.
 	RollbackInProgress RayServiceConditionType = "RollbackInProgress"
+	// RayServiceSuspending means the RayService is in the middle of deleting its owned resources in response to Spec.Suspend.
+	// Once entered, the suspend operation completes atomically regardless of later changes to Spec.Suspend.
+	RayServiceSuspending RayServiceConditionType = "Suspending"
+	// RayServiceSuspended means all resources owned by the RayService controller have been deleted and the RayService is suspended.
+	RayServiceSuspended RayServiceConditionType = "Suspended"
 )
 
 const (
@@ -221,6 +232,9 @@ const (
 	NoActiveCluster                RayServiceConditionReason = "NoActiveCluster"
 	RayServiceValidationFailed     RayServiceConditionReason = "ValidationFailed"
 	TargetClusterChanged           RayServiceConditionReason = "TargetClusterChanged"
+	SuspendRequested               RayServiceConditionReason = "SuspendRequested"
+	SuspendInProgress              RayServiceConditionReason = "SuspendInProgress"
+	SuspendComplete                RayServiceConditionReason = "SuspendComplete"
 )
 
 // +kubebuilder:object:root=true