From af5b4e148a7d41d05b11b44202996573fc974fa9 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Thu, 7 May 2026 11:32:36 -0700 Subject: [PATCH 1/2] Cache image-pull-secret reads in webhook via controller-runtime informer The embedded secret manager's mutating webhook issues two synchronous kube-apiserver Gets per pod admission to resolve image pull secrets (one against the reference namespace, one against the pod namespace for the mirrored copy). At admission scale this is unnecessary load on the API server. Replace the bare controller-runtime client.New with a cache-backed delegating client. The cache is started for the lifetime of the webhook and watches Secrets cluster-wide; reads are served from the informer, writes still go through the API server. Signed-off-by: Haytham Abuelfutuh --- .../secret/secrets_injector.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/flyteplugins/go/tasks/pluginmachinery/secret/secrets_injector.go b/flyteplugins/go/tasks/pluginmachinery/secret/secrets_injector.go index 343a2858bf7..5ae5e35f415 100644 --- a/flyteplugins/go/tasks/pluginmachinery/secret/secrets_injector.go +++ b/flyteplugins/go/tasks/pluginmachinery/secret/secrets_injector.go @@ -6,6 +6,7 @@ import ( corev1 "k8s.io/api/core/v1" k8sRuntime "k8s.io/apimachinery/pkg/runtime" + ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/secret/config" @@ -62,8 +63,27 @@ func newSecretsInjector( return nil, fmt.Errorf("failed to add core v1 to scheme: %w", err) } + secretInformerCache, err := ctrlcache.New(kubeConfig, ctrlcache.Options{ + Scheme: ctrlRuntimeScheme, + }) + if err != nil { + return nil, fmt.Errorf("failed to create informer cache: %w", err) + } + + go func() { + if err := secretInformerCache.Start(ctx); err != nil { + logger.Errorf(ctx, "secret informer cache stopped: %v", err) + } + }() + if !secretInformerCache.WaitForCacheSync(ctx) { + return nil, fmt.Errorf("secret informer cache failed to sync") + } + ctrlRuntimeClient, err := client.New(kubeConfig, client.Options{ Scheme: ctrlRuntimeScheme, + Cache: &client.CacheOptions{ + Reader: secretInformerCache, + }, }) if err != nil { return nil, fmt.Errorf("failed to create controller-runtime client: %w", err) From 319199b4961a090217d845b0012d267108e977a6 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Thu, 7 May 2026 14:08:17 -0700 Subject: [PATCH 2/2] Skip Secret informer when ImagePullSecrets is disabled The k8s client is only used by embedded_secret_manager's image-pull-secret path, which is gated on EmbeddedSecretManagerConfig.ImagePullSecrets.Enabled. When the feature is disabled the informer was generating list/watch traffic for nothing, so only build the cache when the feature is on. In the disabled path fall back to a bare direct client. Also explicitly register the Secret informer with the cache. controller- runtime caches are type-lazy (an informer is only started on first access), so the previous code relied on the embedded secret manager being the only caller. The explicit registration: - documents that this cache is intended only for Secret reads - ensures WaitForCacheSync actually blocks on the initial Secret list rather than returning immediately on an empty informer set, so the first admission isn't a guaranteed cache miss Signed-off-by: Haytham Abuelfutuh --- .../secret/secrets_injector.go | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/flyteplugins/go/tasks/pluginmachinery/secret/secrets_injector.go b/flyteplugins/go/tasks/pluginmachinery/secret/secrets_injector.go index 5ae5e35f415..3edb6dfbca8 100644 --- a/flyteplugins/go/tasks/pluginmachinery/secret/secrets_injector.go +++ b/flyteplugins/go/tasks/pluginmachinery/secret/secrets_injector.go @@ -63,28 +63,38 @@ func newSecretsInjector( return nil, fmt.Errorf("failed to add core v1 to scheme: %w", err) } - secretInformerCache, err := ctrlcache.New(kubeConfig, ctrlcache.Options{ - Scheme: ctrlRuntimeScheme, - }) - if err != nil { - return nil, fmt.Errorf("failed to create informer cache: %w", err) - } + // The k8s client is only used by embedded_secret_manager's image-pull-secret + // path. When that feature is disabled we use a direct client and skip the + // informer entirely; otherwise we back the client with a Secret informer. + clientOpts := client.Options{Scheme: ctrlRuntimeScheme} + if webhookConfig.EmbeddedSecretManagerConfig.ImagePullSecrets.Enabled { + secretInformerCache, err := ctrlcache.New(kubeConfig, ctrlcache.Options{ + Scheme: ctrlRuntimeScheme, + }) + if err != nil { + return nil, fmt.Errorf("failed to create informer cache: %w", err) + } - go func() { - if err := secretInformerCache.Start(ctx); err != nil { - logger.Errorf(ctx, "secret informer cache stopped: %v", err) + // Explicitly register the Secret informer so the cache only watches + // Secrets (not every kind in corev1) and so WaitForCacheSync below + // actually blocks on the initial Secret list. + if _, err := secretInformerCache.GetInformer(ctx, &corev1.Secret{}); err != nil { + return nil, fmt.Errorf("failed to register Secret informer: %w", err) } - }() - if !secretInformerCache.WaitForCacheSync(ctx) { - return nil, fmt.Errorf("secret informer cache failed to sync") + + go func() { + if err := secretInformerCache.Start(ctx); err != nil { + logger.Errorf(ctx, "secret informer cache stopped: %v", err) + } + }() + if !secretInformerCache.WaitForCacheSync(ctx) { + return nil, fmt.Errorf("secret informer cache failed to sync") + } + + clientOpts.Cache = &client.CacheOptions{Reader: secretInformerCache} } - ctrlRuntimeClient, err := client.New(kubeConfig, client.Options{ - Scheme: ctrlRuntimeScheme, - Cache: &client.CacheOptions{ - Reader: secretInformerCache, - }, - }) + ctrlRuntimeClient, err := client.New(kubeConfig, clientOpts) if err != nil { return nil, fmt.Errorf("failed to create controller-runtime client: %w", err) }