diff --git a/apis/projectcontour/v1alpha1/contourdeployment.go b/apis/projectcontour/v1alpha1/contourdeployment.go index 33a04956fb2..37c37590e65 100644 --- a/apis/projectcontour/v1alpha1/contourdeployment.go +++ b/apis/projectcontour/v1alpha1/contourdeployment.go @@ -237,6 +237,16 @@ type EnvoySettings struct { // +optional BaseID int32 `json:"baseID,omitempty"` + // Concurrency specifies the number of worker threads to run for Envoy. + // If not specified, Envoy defaults to the number of hardware threads on + // the machine. Setting this to a lower value on high-core-count machines + // avoids excessive worker threads and reduces idle CPU and Memory usage. + // defaults to 0 (use Envoy's default behavior). + // + // +kubebuilder:validation:Minimum=0 + // +optional + Concurrency int32 `json:"concurrency,omitempty"` + // OverloadMaxHeapSize defines the maximum heap memory of the envoy controlled by the overload manager. // When the value is greater than 0, the overload manager is enabled, // and when envoy reaches 95% of the maximum heap size, it performs a shrink heap operation, diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml index 97bfb59ce30..7ab673d4f74 100644 --- a/examples/contour/01-crds.yaml +++ b/examples/contour/01-crds.yaml @@ -1720,6 +1720,16 @@ spec: format: int32 minimum: 0 type: integer + concurrency: + description: |- + Concurrency specifies the number of worker threads to run for Envoy. + If not specified, Envoy defaults to the number of hardware threads on + the machine. Setting this to a lower value on high-core-count machines + avoids excessive worker threads and reduces idle CPU and Memory usage. + defaults to 0 (use Envoy's default behavior). + format: int32 + minimum: 0 + type: integer daemonSet: description: |- DaemonSet describes the settings for running envoy as a `DaemonSet`. diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml index bbfe8363101..3f24a10a0ea 100644 --- a/examples/render/contour-deployment.yaml +++ b/examples/render/contour-deployment.yaml @@ -1939,6 +1939,16 @@ spec: format: int32 minimum: 0 type: integer + concurrency: + description: |- + Concurrency specifies the number of worker threads to run for Envoy. + If not specified, Envoy defaults to the number of hardware threads on + the machine. Setting this to a lower value on high-core-count machines + avoids excessive worker threads and reduces idle CPU and Memory usage. + defaults to 0 (use Envoy's default behavior). + format: int32 + minimum: 0 + type: integer daemonSet: description: |- DaemonSet describes the settings for running envoy as a `DaemonSet`. diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml index b2ba7c0f7a1..3b97cadf007 100644 --- a/examples/render/contour-gateway-provisioner.yaml +++ b/examples/render/contour-gateway-provisioner.yaml @@ -1731,6 +1731,16 @@ spec: format: int32 minimum: 0 type: integer + concurrency: + description: |- + Concurrency specifies the number of worker threads to run for Envoy. + If not specified, Envoy defaults to the number of hardware threads on + the machine. Setting this to a lower value on high-core-count machines + avoids excessive worker threads and reduces idle CPU and Memory usage. + defaults to 0 (use Envoy's default behavior). + format: int32 + minimum: 0 + type: integer daemonSet: description: |- DaemonSet describes the settings for running envoy as a `DaemonSet`. diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml index e31713e8cf9..13d48e43099 100644 --- a/examples/render/contour-gateway.yaml +++ b/examples/render/contour-gateway.yaml @@ -1756,6 +1756,16 @@ spec: format: int32 minimum: 0 type: integer + concurrency: + description: |- + Concurrency specifies the number of worker threads to run for Envoy. + If not specified, Envoy defaults to the number of hardware threads on + the machine. Setting this to a lower value on high-core-count machines + avoids excessive worker threads and reduces idle CPU and Memory usage. + defaults to 0 (use Envoy's default behavior). + format: int32 + minimum: 0 + type: integer daemonSet: description: |- DaemonSet describes the settings for running envoy as a `DaemonSet`. diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml index 72e156e5d60..9625732e92b 100644 --- a/examples/render/contour.yaml +++ b/examples/render/contour.yaml @@ -1939,6 +1939,16 @@ spec: format: int32 minimum: 0 type: integer + concurrency: + description: |- + Concurrency specifies the number of worker threads to run for Envoy. + If not specified, Envoy defaults to the number of hardware threads on + the machine. Setting this to a lower value on high-core-count machines + avoids excessive worker threads and reduces idle CPU and Memory usage. + defaults to 0 (use Envoy's default behavior). + format: int32 + minimum: 0 + type: integer daemonSet: description: |- DaemonSet describes the settings for running envoy as a `DaemonSet`. diff --git a/internal/provisioner/controller/gateway.go b/internal/provisioner/controller/gateway.go index 5aab0024283..c9626a55ef9 100644 --- a/internal/provisioner/controller/gateway.go +++ b/internal/provisioner/controller/gateway.go @@ -351,6 +351,10 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct contourModel.Spec.EnvoyBaseID = envoyParams.BaseID } + if envoyParams.Concurrency > 0 { + contourModel.Spec.EnvoyConcurrency = envoyParams.Concurrency + } + if envoyParams.OverloadMaxHeapSize > 0 { contourModel.Spec.EnvoyMaxHeapSizeBytes = envoyParams.OverloadMaxHeapSize } diff --git a/internal/provisioner/controller/gateway_test.go b/internal/provisioner/controller/gateway_test.go index 2f6692e6c6d..caac7769f0a 100644 --- a/internal/provisioner/controller/gateway_test.go +++ b/internal/provisioner/controller/gateway_test.go @@ -1128,6 +1128,32 @@ func TestGatewayReconcile(t *testing.T) { }, }, + "If ContourDeployment.Spec.Envoy.Concurrency is specified, the Envoy container's arguments contain --concurrency": { + gatewayClass: reconcilableGatewayClassWithParams("gatewayclass-1", controller), + gatewayClassParams: &contour_v1alpha1.ContourDeployment{ + ObjectMeta: meta_v1.ObjectMeta{ + Namespace: "projectcontour", + Name: "gatewayclass-1-params", + }, + Spec: contour_v1alpha1.ContourDeploymentSpec{ + Envoy: &contour_v1alpha1.EnvoySettings{ + Concurrency: 4, + }, + }, + }, + gateway: makeGateway(), + assertions: func(t *testing.T, r *gatewayReconciler, _ *gatewayapi_v1.Gateway, _ error) { + ds := &apps_v1.DaemonSet{ + ObjectMeta: meta_v1.ObjectMeta{ + Namespace: "gateway-1", + Name: "envoy-gateway-1", + }, + } + require.NoError(t, r.client.Get(context.Background(), keyFor(ds), ds)) + assert.Contains(t, ds.Spec.Template.Spec.Containers[1].Args, "--concurrency 4") + }, + }, + "If ContourDeployment.Spec.Envoy.OverloadMaxHeapSize is specified, the envoy-initconfig container's arguments contain --overload-max-heap": { gatewayClass: reconcilableGatewayClassWithParams("gatewayclass-1", controller), gatewayClassParams: &contour_v1alpha1.ContourDeployment{ diff --git a/internal/provisioner/model/model.go b/internal/provisioner/model/model.go index 4dbc3594184..2928b0ab24d 100644 --- a/internal/provisioner/model/model.go +++ b/internal/provisioner/model/model.go @@ -48,6 +48,7 @@ func Default(namespace, name string) *Contour { EnvoyReplicas: 2, // ignored if not provisioning Envoy as a deployment. EnvoyLogLevel: contour_v1alpha1.InfoLog, EnvoyBaseID: 0, + EnvoyConcurrency: 0, EnvoyMaxHeapSizeBytes: 0, EnvoyMaxDownstreamConnections: 0, NetworkPublishing: NetworkPublishing{ @@ -245,6 +246,10 @@ type ContourSpec struct { // defaults to 0. EnvoyBaseID int32 + // EnvoyConcurrency specifies the number of worker threads for Envoy. + // defaults to 0 (use Envoy's default behavior). + EnvoyConcurrency int32 + // EnvoyMaxHeapSizeBytes defines how much memory the overload manager controls Envoy to allocate at most. // defaults to 0. EnvoyMaxHeapSizeBytes uint64 diff --git a/internal/provisioner/objects/dataplane/dataplane.go b/internal/provisioner/objects/dataplane/dataplane.go index 4d223f0e066..f4cbca620f7 100644 --- a/internal/provisioner/objects/dataplane/dataplane.go +++ b/internal/provisioner/objects/dataplane/dataplane.go @@ -141,6 +141,21 @@ func EnsureDataPlaneDeleted(ctx context.Context, cli client.Client, contour *mod return objects.EnsureObjectDeleted(ctx, cli, deployObj, contour) } +func envoyArgs(contour *model.Contour) []string { + args := []string{ + "-c", + filepath.Join("/", envoyCfgVolMntDir, envoyCfgFileName), + fmt.Sprintf("--service-cluster $(%s)", envoyNsEnvVar), + fmt.Sprintf("--service-node $(%s)", envoyPodEnvVar), + fmt.Sprintf("--log-level %s", contour.Spec.EnvoyLogLevel), + fmt.Sprintf("--base-id %d", contour.Spec.EnvoyBaseID), + } + if contour.Spec.EnvoyConcurrency > 0 { + args = append(args, fmt.Sprintf("--concurrency %d", contour.Spec.EnvoyConcurrency)) + } + return args +} + func desiredContainers(contour *model.Contour, contourImage, envoyImage string) ([]core_v1.Container, []core_v1.Container) { var ( metricsPort = objects.EnvoyMetricsPort @@ -204,14 +219,7 @@ func desiredContainers(contour *model.Contour, contourImage, envoyImage string) Command: []string{ "envoy", }, - Args: []string{ - "-c", - filepath.Join("/", envoyCfgVolMntDir, envoyCfgFileName), - fmt.Sprintf("--service-cluster $(%s)", envoyNsEnvVar), - fmt.Sprintf("--service-node $(%s)", envoyPodEnvVar), - fmt.Sprintf("--log-level %s", contour.Spec.EnvoyLogLevel), - fmt.Sprintf("--base-id %d", contour.Spec.EnvoyBaseID), - }, + Args: envoyArgs(contour), Env: []core_v1.EnvVar{ { Name: envoyNsEnvVar, diff --git a/internal/provisioner/objects/dataplane/dataplane_test.go b/internal/provisioner/objects/dataplane/dataplane_test.go index c71b99c11d8..21c12b5d1f2 100644 --- a/internal/provisioner/objects/dataplane/dataplane_test.go +++ b/internal/provisioner/objects/dataplane/dataplane_test.go @@ -312,6 +312,7 @@ func TestDesiredDaemonSet(t *testing.T) { testEnvoyImage := "docker.io/envoyproxy/envoy:test" testLogLevelArg := "--log-level debug" testBaseIDArg := "--base-id 1" + testConcurrencyArg := "--concurrency 4" testEnvoyMaxHeapSize := "--overload-max-heap=8000000000" testEnvoyMaxDownstreamConn := "--overload-downstream-max-conn=42" @@ -338,6 +339,8 @@ func TestDesiredDaemonSet(t *testing.T) { } // Change the Envoy base id to test --base-id 1 cntr.Spec.EnvoyBaseID = 1 + // Change the Envoy concurrency to test --concurrency 4 + cntr.Spec.EnvoyConcurrency = 4 cntr.Spec.EnvoyMaxHeapSizeBytes = 8000000000 cntr.Spec.EnvoyMaxDownstreamConnections = 42 @@ -346,6 +349,7 @@ func TestDesiredDaemonSet(t *testing.T) { container := checkDaemonSetHasContainer(t, ds, EnvoyContainerName, true) checkContainerHasArg(t, container, testLogLevelArg) checkContainerHasArg(t, container, testBaseIDArg) + checkContainerHasArg(t, container, testConcurrencyArg) checkContainerHasImage(t, container, testEnvoyImage) checkContainerHasReadinessPort(t, container, 8002) diff --git a/site/content/docs/main/config/api-reference.html b/site/content/docs/main/config/api-reference.html index 9d28f0210d4..1edbcdcdf06 100644 --- a/site/content/docs/main/config/api-reference.html +++ b/site/content/docs/main/config/api-reference.html @@ -7477,6 +7477,23 @@

EnvoySettings +concurrency +
+ +int32 + + + +(Optional) +

Concurrency specifies the number of worker threads to run for Envoy. +If not specified, Envoy defaults to the number of hardware threads on +the machine. Setting this to a lower value on high-core-count machines +avoids excessive worker threads and reduces idle CPU and Memory usage. +defaults to 0 (use Envoy’s default behavior).

+ + + + overloadMaxHeapSize