support nlb dryrun & ignore clb reuse key dryrun check

cmd/manager/main.go

@@ -14,6 +14,7 @@ import (
 	prvd "k8s.io/cloud-provider-alibaba-cloud/pkg/provider"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/alibaba"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/dryrun"
+	utildryrun "k8s.io/cloud-provider-alibaba-cloud/pkg/util/dryrun"
 	"k8s.io/cloud-provider-alibaba-cloud/version"
 	"k8s.io/klog/v2/klogr"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -98,6 +99,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	if ctrlCfg.ControllerCFG.DryRun {
+		go func() {
+			<-utildryrun.Done()
+			log.Info("ccm initial process finished.")
+			err := utildryrun.ResultEvent(mgr.GetClient(), utildryrun.SUCCESS, "ccm initial process finished")
+			if err != nil {
+				log.Error(err, "write precheck event failed")
+			}
+			os.Exit(0)
+		}()
+	}
+
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		log.Error(err, "Manager exited non-zero: %s", err.Error())
 		os.Exit(1)

pkg/controller/service/clbv1/dryrun.go

@@ -2,13 +2,14 @@ package clbv1
 
 import (
 	"context"
+	"os"
+	"sync"
+
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/dryrun"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/util"
+	"k8s.io/cloud-provider-alibaba-cloud/pkg/util/dryrun"
 	"k8s.io/klog/v2"
-	"os"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sync"
 )
 
 var initial = sync.Map{}
@@ -31,11 +32,7 @@ func initMap(client client.Client) {
 	}
 	util.ServiceLog.Info("ccm initial process finished.", "length", length)
 	if length == 0 {
-		err := dryrun.ResultEvent(client, dryrun.SUCCESS, "ccm initial process finished")
-		if err != nil {
-			util.ServiceLog.Error(err, "fail to write precheck event")
-		}
-		os.Exit(0)
+		dryrun.Finish(dryrun.SLB)
 	}
 }
 

pkg/controller/service/clbv1/listeners.go

@@ -17,8 +17,8 @@ import (
 	"strconv"
 	"strings"
 
-	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/dryrun"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/util"
+	"k8s.io/cloud-provider-alibaba-cloud/pkg/util/dryrun"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"

pkg/controller/service/clbv1/service_controller.go

@@ -5,12 +5,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
 	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/cloud-provider-alibaba-cloud/pkg/util/dryrun"
 
 	"github.com/go-logr/logr"
 	"golang.org/x/time/rate"
@@ -42,11 +42,13 @@ import (
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/controller/helper"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/model"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/alibaba/vpc"
-	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/dryrun"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/util"
 )
 
 func Add(mgr manager.Manager, ctx *shared.SharedContext) error {
+	if ctrlCfg.ControllerCFG.DryRun {
+		dryrun.RegisterDryRun(dryrun.SLB)
+	}
 	r, err := newReconciler(mgr, ctx)
 	if err != nil {
 		return err
@@ -116,6 +118,10 @@ func add(mgr manager.Manager, r *ReconcileService) error {
 		return fmt.Errorf("watch resource svc error: %s", err.Error())
 	}
 
+	if ctrlCfg.ControllerCFG.DryRun {
+		return mgr.Add(&serviceController{c: c, recon: r})
+	}
+
 	if utilfeature.DefaultFeatureGate.Enabled(ctrlCfg.EndpointSlice) {
 		// watch endpointslice
 		if err := c.Watch(source.Kind(mgr.GetCache(), &discovery.EndpointSlice{}),
@@ -169,12 +175,8 @@ func (m *ReconcileService) reconcile(c context.Context, request reconcile.Reques
 			initial.Store(request.String(), 1)
 			util.ServiceLog.Info("DryRun: reconcile finished", "service", request.NamespacedName.String())
 			if mapfull() {
-				util.ServiceLog.Info("ccm initial process finished.")
-				err := dryrun.ResultEvent(m.kubeClient, dryrun.SUCCESS, "ccm initial process finished")
-				if err != nil {
-					util.ServiceLog.Error(err, "write precheck event failed", "service", request.NamespacedName.String())
-				}
-				os.Exit(0)
+				util.ServiceLog.Info("ccm slb dryrun process finished")
+				dryrun.Finish(dryrun.SLB)
 			}
 			err = nil
 		}

pkg/controller/service/clbv1/vgroups_test.go

@@ -11,7 +11,7 @@ import (
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/controller/service/reconcile/backend"
 	svcCtx "k8s.io/cloud-provider-alibaba-cloud/pkg/controller/service/reconcile/context"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/model"
-	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/dryrun"
+	"k8s.io/cloud-provider-alibaba-cloud/pkg/util/dryrun"
 	"k8s.io/klog/v2/klogr"
 )
 

pkg/controller/service/nlbv2/dryrun.go

@@ -0,0 +1,57 @@
+package nlbv2
+
+import (
+	"context"
+	"os"
+	"sync"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/cloud-provider-alibaba-cloud/pkg/util"
+	"k8s.io/cloud-provider-alibaba-cloud/pkg/util/dryrun"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var initial = sync.Map{}
+
+func initMap(client client.Client) {
+	svcs := v1.ServiceList{}
+	err := client.List(context.TODO(), &svcs)
+	if err != nil {
+		klog.Infof("init Map error: %s", err.Error())
+		os.Exit(1)
+	}
+
+	length := 0
+	for _, m := range svcs.Items {
+		if !needAdd(&m) {
+			continue
+		}
+		length++
+		initial.Store(util.NamespacedName(&m).String(), 0)
+	}
+	util.NLBLog.Info("ccm initial process finished.", "length", length)
+	if length == 0 {
+		dryrun.Finish(dryrun.NLB)
+	}
+}
+
+func mapfull() bool {
+	total, unsync := 0, 0
+	initial.Range(
+		func(key, value interface{}) bool {
+			val, ok := value.(int)
+			if !ok {
+				// not supposed
+				return true
+			}
+			if val != 1 {
+				unsync += 1
+			}
+			total += 1
+			return true
+		},
+	)
+	util.NLBLog.Info("Reconcile process", "total", total, "unsync", unsync)
+	return unsync == 0
+}

pkg/controller/service/nlbv2/dryrun_test.go

@@ -0,0 +1,23 @@
+package nlbv2
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMapFull(t *testing.T) {
+	assert.Equal(t, mapfull(), true)
+
+	initial.Store("default/test-svc", 0)
+	assert.Equal(t, mapfull(), false)
+
+	initial.Store("default/test-svc", 1)
+	assert.Equal(t, mapfull(), true)
+}
+
+//func TestInitMap(t *testing.T) {
+//	initMap(getFakeKubeClient())
+//	_, ok := initial.Load(fmt.Sprintf("%s/%s", NS, SvcName))
+//	assert.Equal(t, ok, true)
+//}

pkg/controller/service/nlbv2/listeners.go

@@ -22,8 +22,8 @@ import (
 	prvd "k8s.io/cloud-provider-alibaba-cloud/pkg/provider"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/alibaba/base"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/alibaba/nlb"
-	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/dryrun"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/util"
+	"k8s.io/cloud-provider-alibaba-cloud/pkg/util/dryrun"
 	"k8s.io/klog/v2"
 )
 

pkg/controller/service/nlbv2/model_applier.go

@@ -14,8 +14,8 @@ import (
 	nlbmodel "k8s.io/cloud-provider-alibaba-cloud/pkg/model/nlb"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/model/tag"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/alibaba/base"
-	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/dryrun"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/util"
+	"k8s.io/cloud-provider-alibaba-cloud/pkg/util/dryrun"
 )
 
 type serverGroupApplyResult struct {

pkg/controller/service/nlbv2/nlb_controller.go

@@ -27,8 +27,8 @@ import (
 	svcCtx "k8s.io/cloud-provider-alibaba-cloud/pkg/controller/service/reconcile/context"
 	nlbmodel "k8s.io/cloud-provider-alibaba-cloud/pkg/model/nlb"
 	prvd "k8s.io/cloud-provider-alibaba-cloud/pkg/provider"
-	"k8s.io/cloud-provider-alibaba-cloud/pkg/provider/dryrun"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/util"
+	"k8s.io/cloud-provider-alibaba-cloud/pkg/util/dryrun"
 	"k8s.io/cloud-provider-alibaba-cloud/pkg/util/metric"
 	"k8s.io/klog/v2"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
@@ -41,6 +41,9 @@ import (
 )
 
 func Add(mgr manager.Manager, ctx *shared.SharedContext) error {
+	if ctrlCfg.ControllerCFG.DryRun {
+		dryrun.RegisterDryRun(dryrun.NLB)
+	}
 	reconciler, err := newReconciler(mgr, ctx)
 	if err != nil {
 		return fmt.Errorf("new nlb reconciler error: %s", err.Error())
@@ -75,6 +78,9 @@ type nlbController struct {
 }
 
 func (n nlbController) Start(ctx context.Context) error {
+	if ctrlCfg.ControllerCFG.DryRun {
+		initMap(n.recon.kubeClient)
+	}
 	return n.c.Start(ctx)
 }
 
@@ -104,6 +110,10 @@ func add(mgr manager.Manager, r *ReconcileNLB) error {
 		return fmt.Errorf("watch resource svc error: %s", err.Error())
 	}
 
+	if ctrlCfg.ControllerCFG.DryRun {
+		return mgr.Add(&nlbController{c: c, recon: r})
+	}
+
 	if utilfeature.DefaultFeatureGate.Enabled(ctrlCfg.EndpointSlice) {
 		// watch endpointslice
 		if err := c.Watch(source.Kind(mgr.GetCache(), &discovery.EndpointSlice{}),
@@ -148,15 +158,27 @@ func (m *ReconcileNLB) Reconcile(ctx context.Context, request reconcile.Request)
 	return util.HandleReconcileResult(request, m.reconcile(ctx, request))
 }
 
-func (m *ReconcileNLB) reconcile(c context.Context, request reconcile.Request) error {
+func (m *ReconcileNLB) reconcile(c context.Context, request reconcile.Request) (err error) {
 	startTime := time.Now()
 
+	defer func() {
+		if ctrlCfg.ControllerCFG.DryRun {
+			initial.Store(request.String(), 1)
+			util.NLBLog.Info("DryRun: reconcile finished", "service", request.NamespacedName.String())
+			if mapfull() {
+				util.NLBLog.Info("ccm nlb dryrun process finished")
+				dryrun.Finish(dryrun.NLB)
+			}
+			err = nil
+		}
+	}()
+
 	reconcileID := controller.ReconcileIDFromContext(c)
 	nlbLog := util.NLBLog.WithValues("service", request.NamespacedName.String(), "reconcileID", reconcileID)
 	nlbLog.Info("starting reconcile service")
 
 	svc := &v1.Service{}
-	err := m.kubeClient.Get(context.Background(), request.NamespacedName, svc)
+	err = m.kubeClient.Get(context.Background(), request.NamespacedName, svc)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
 			m.logger.Info("service not found, skip", "service", request.NamespacedName)
@@ -239,7 +261,6 @@ func (m *ReconcileNLB) cleanupLoadBalancerResources(reqCtx *svcCtx.RequestContex
 }
 
 func (m *ReconcileNLB) reconcileLoadBalancerResources(req *svcCtx.RequestContext) error {
-
 	if err := m.finalizerManager.AddFinalizers(req.Ctx, req.Service, helper.NLBFinalizer); err != nil {
 		m.record.Event(req.Service, v1.EventTypeWarning, helper.FailedAddFinalizer,
 			fmt.Sprintf("Error adding finalizer: %s", err.Error()))

pkg/provider/alibaba/nlb/cert.go

@@ -61,7 +61,7 @@ func (p *NLBProvider) AssociateAdditionalCertificatesWithNLBListener(ctx context
 	klog.V(5).Infof("RequestId: %s, API: %s", tea.StringValue(resp.Body.RequestId), "AssociateAdditionalCertificatesWithListener")
 	jobId := tea.StringValue(resp.Body.JobId)
 	if jobId != "" {
-		return p.WaitJobFinish("AssociateAdditionalCertificatesWithListener", jobId)
+		return p.WaitJobFinish(nil, "AssociateAdditionalCertificatesWithListener", jobId)
 	}
 
 	return nil
@@ -81,7 +81,7 @@ func (p *NLBProvider) DisassociateAdditionalCertificatesWithNLBListener(ctx cont
 
 	jobId := tea.StringValue(resp.Body.JobId)
 	if jobId != "" {
-		return p.WaitJobFinish("DisassociateAdditionalCertificatesWithListener", jobId)
+		return p.WaitJobFinish(nil, "DisassociateAdditionalCertificatesWithListener", jobId)
 	}
 	return nil
 }

pkg/provider/alibaba/nlb/listener.go

@@ -116,23 +116,23 @@ func (p *NLBProvider) CreateNLBListener(ctx context.Context, lbId string, lis *n
 	if err != nil {
 		return err
 	}
-	return p.WaitJobFinish("CreateListener", jobId)
+	return p.WaitJobFinish(nil, "CreateListener", jobId)
 }
 
 func (p *NLBProvider) UpdateNLBListener(ctx context.Context, lis *nlbmodel.ListenerAttribute) error {
 	jobId, err := p.UpdateNLBListenerAsync(ctx, lis)
 	if err != nil {
 		return err
 	}
-	return p.WaitJobFinish("UpdateListener", jobId)
+	return p.WaitJobFinish(nil, "UpdateListener", jobId)
 }
 
 func (p *NLBProvider) DeleteNLBListener(ctx context.Context, listenerId string) error {
 	jobId, err := p.DeleteNLBListenerAsync(ctx, listenerId)
 	if err != nil {
 		return err
 	}
-	return p.WaitJobFinish("DeleteListener", jobId)
+	return p.WaitJobFinish(nil, "DeleteListener", jobId)
 }
 
 func (p *NLBProvider) StartNLBListener(ctx context.Context, listenerId string) error {