diff --git a/api/v1beta1/consts.go b/api/v1beta1/consts.go index c6fa7ca8654..8a6931853e9 100644 --- a/api/v1beta1/consts.go +++ b/api/v1beta1/consts.go @@ -130,6 +130,8 @@ const ( NetworkInterfaceReadyCondition clusterv1beta1.ConditionType = "NetworkInterfacesReady" // PrivateEndpointsReadyCondition means the private endpoints exist and are ready to be used. PrivateEndpointsReadyCondition clusterv1beta1.ConditionType = "PrivateEndpointsReady" + // PrivateLinksReadyCondition means the private links exist and are ready to be used. + PrivateLinksReadyCondition clusterv1beta1.ConditionType = "PrivateLinksReady" // FleetReadyCondition means the Fleet exists and is ready to be used. FleetReadyCondition clusterv1beta1.ConditionType = "FleetReady" // AKSExtensionsReadyCondition means the AKS Extensions exist and are ready to be used. diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 7a5344ca25b..e89b4a4863c 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -363,6 +363,9 @@ type LoadBalancerSpec struct { // FrontendIPsCount specifies the number of frontend IP addresses for the load balancer. // +optional FrontendIPsCount *int32 `json:"frontendIPsCount,omitempty"` + // PrivateLinks to the load balancer (max 8 private links). + // +optional + PrivateLinks []PrivateLink `json:"privateLinks,omitempty"` // BackendPool describes the backend pool of the load balancer. // +optional BackendPool BackendPool `json:"backendPool,omitempty"` @@ -415,6 +418,57 @@ type IPTag struct { Tag string `json:"tag"` } +// PrivateLink configures an Azure private link. +type PrivateLink struct { + // Name of the private link. + // +optional + Name string `json:"name,omitempty"` + + // NATIPConfigurations specify up to 8 NAT IP configurations for the private link. + NATIPConfigurations []PrivateLinkNATIPConfiguration `json:"natIPConfigurations"` + + // LBFrontendIPConfigNames are the names of the load balancer FrontendIP to which the private link will forward + // requests. The specified frontend IP configs must have the private IP set. + LBFrontendIPConfigNames []string `json:"lbFrontendIPConfigNames"` + + // AllowedSubscriptions is a list of subscriptions from which the private link can be accessed. + // +optional + AllowedSubscriptions []*string `json:"allowedSubscriptions,omitempty"` + + // AutoApprovedSubscriptions is a list of subscription for which the connections to private link are automatically + // approved. + // +optional + AutoApprovedSubscriptions []*string `json:"autoApprovedSubscriptions,omitempty"` + + // EnableProxyProtocol indicates whether the private link service is enabled for proxy protocol or not. + // +optional + EnableProxyProtocol *bool `json:"enableProxyProtocol,omitempty"` +} + +// PrivateLinkNATIPConfiguration specifies NAT IP configuration for the private link. +type PrivateLinkNATIPConfiguration struct { + // AllocationMethod specifies how the private link NAT IPs are allocated: "Static" or "Dynamic". + AllocationMethod string `json:"allocationMethod"` + + // Subnet from which the IP is allocated. + Subnet string `json:"subnet"` + + // +optional + PrivateIPAddress string `json:"privateIPAddress,omitempty"` +} + +// PrivateLinkNATIPAllocationMethod specifies whether the private link NAT IPs are allocated statically or dynamically. +// +kubebuilder:validation:Enum=Static;Dynamic +type PrivateLinkNATIPAllocationMethod string + +const ( + // NATIPAllocationMethodStatic specifies that NAT IP for private link is allocated statically (manually set by user). + NATIPAllocationMethodStatic PrivateLinkNATIPAllocationMethod = "Static" + + // NATIPAllocationMethodDynamic specifies that NAT IP for private link is allocated dynamically (by Azure). + NATIPAllocationMethodDynamic PrivateLinkNATIPAllocationMethod = "Dynamic" +) + // VMState describes the state of an Azure virtual machine. // // Deprecated: use ProvisioningState. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 3ab3aa92d1f..608aaad82cd 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -3453,6 +3453,13 @@ func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = new(int32) **out = **in } + if in.PrivateLinks != nil { + in, out := &in.PrivateLinks, &out.PrivateLinks + *out = make([]PrivateLink, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } out.BackendPool = in.BackendPool in.LoadBalancerClassSpec.DeepCopyInto(&out.LoadBalancerClassSpec) } @@ -3998,6 +4005,73 @@ func (in PrivateEndpoints) DeepCopy() PrivateEndpoints { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrivateLink) DeepCopyInto(out *PrivateLink) { + *out = *in + if in.NATIPConfigurations != nil { + in, out := &in.NATIPConfigurations, &out.NATIPConfigurations + *out = make([]PrivateLinkNATIPConfiguration, len(*in)) + copy(*out, *in) + } + if in.LBFrontendIPConfigNames != nil { + in, out := &in.LBFrontendIPConfigNames, &out.LBFrontendIPConfigNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AllowedSubscriptions != nil { + in, out := &in.AllowedSubscriptions, &out.AllowedSubscriptions + *out = make([]*string, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(string) + **out = **in + } + } + } + if in.AutoApprovedSubscriptions != nil { + in, out := &in.AutoApprovedSubscriptions, &out.AutoApprovedSubscriptions + *out = make([]*string, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(string) + **out = **in + } + } + } + if in.EnableProxyProtocol != nil { + in, out := &in.EnableProxyProtocol, &out.EnableProxyProtocol + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrivateLink. +func (in *PrivateLink) DeepCopy() *PrivateLink { + if in == nil { + return nil + } + out := new(PrivateLink) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrivateLinkNATIPConfiguration) DeepCopyInto(out *PrivateLinkNATIPConfiguration) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrivateLinkNATIPConfiguration. +func (in *PrivateLinkNATIPConfiguration) DeepCopy() *PrivateLinkNATIPConfiguration { + if in == nil { + return nil + } + out := new(PrivateLinkNATIPConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PrivateLinkServiceConnection) DeepCopyInto(out *PrivateLinkServiceConnection) { *out = *in diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index d057c254d1e..fd9f1cc257a 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -47,6 +47,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure/services/natgateways" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privatedns" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/privatelinks" "sigs.k8s.io/cluster-api-provider-azure/azure/services/publicips" "sigs.k8s.io/cluster-api-provider-azure/azure/services/routetables" "sigs.k8s.io/cluster-api-provider-azure/azure/services/securitygroups" @@ -452,8 +453,16 @@ func (s *ClusterScope) SubnetSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1 numberOfSubnets++ } - subnetSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1api20201101.VirtualNetworksSubnet], 0, numberOfSubnets) + ipConfigSubnetNames := make(map[string]struct{}) + if apiServerLB := s.AzureCluster.Spec.NetworkSpec.APIServerLB; apiServerLB != nil { + for _, privateLink := range s.AzureCluster.Spec.NetworkSpec.APIServerLB.PrivateLinks { + for _, ipConfig := range privateLink.NATIPConfigurations { + ipConfigSubnetNames[ipConfig.Subnet] = struct{}{} + } + } + } + subnetSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1api20201101.VirtualNetworksSubnet], 0, numberOfSubnets) for _, subnet := range s.AzureCluster.Spec.NetworkSpec.Subnets { subnetSpec := &subnets.SubnetSpec{ Name: subnet.Name, @@ -468,6 +477,11 @@ func (s *ClusterScope) SubnetSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1 NatGatewayName: subnet.NatGateway.Name, ServiceEndpoints: subnet.ServiceEndpoints, } + // Check if subnet is used for the private link NAT IP + if _, ok := ipConfigSubnetNames[subnet.Name]; ok { + subnetSpec.UsedForPrivateLinkNATIP = true + } + subnetSpecs = append(subnetSpecs, subnetSpec) } @@ -1117,6 +1131,11 @@ func (s *ClusterScope) GetLongRunningOperationState(name, service, futureType st return futures.Get(s.AzureCluster, name, service, futureType) } +// GetLongRunningOperationStates will get the specified futures on the AzureCluster status. +func (s *ClusterScope) GetLongRunningOperationStates(service, futureType string) infrav1.Futures { + return futures.GetByServiceAndType(s.AzureCluster, service, futureType) +} + // DeleteLongRunningOperationState will delete the future from the AzureCluster status. func (s *ClusterScope) DeleteLongRunningOperationState(name, service, futureType string) { futures.Delete(s.AzureCluster, name, service, futureType) @@ -1237,7 +1256,41 @@ func (s *ClusterScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*aso return privateEndpointSpecs } -func (s *ClusterScope) getLastAppliedSecurityRules(nsgName string) map[string]any { +// PrivateLinkSpecs returns the private link specs. +func (s *ClusterScope) PrivateLinkSpecs() []azure.ResourceSpecGetter { + // First we get all private links to API server load balancer. + // Other load balancers (ControlPlaneOutboundLB and NodeOutboundLB) are outbound, so we cannot create private links + // for those. + privateLinks := s.AzureCluster.Spec.NetworkSpec.APIServerLB.PrivateLinks + privateLinksSpecs := make([]azure.ResourceSpecGetter, 0, len(privateLinks)) + + for _, privateLink := range privateLinks { + privateLinkSpec := privatelinks.PrivateLinkSpec{ + Name: privateLink.Name, + ResourceGroup: s.ResourceGroup(), + SubscriptionID: s.SubscriptionID(), + Location: s.Location(), + VNetResourceGroup: s.Vnet().ResourceGroup, + VNet: s.Vnet().Name, + LoadBalancerName: s.APIServerLBName(), + LBFrontendIPConfigNames: privateLink.LBFrontendIPConfigNames, + AllowedSubscriptions: privateLink.AllowedSubscriptions, + AutoApprovedSubscriptions: privateLink.AutoApprovedSubscriptions, + EnableProxyProtocol: privateLink.EnableProxyProtocol, + ClusterName: s.ClusterName(), + AdditionalTags: s.AdditionalTags(), + } + // Set NAT IP configuration + for _, natIPConfiguration := range privateLink.NATIPConfigurations { + privateLinkSpec.NATIPConfiguration = append(privateLinkSpec.NATIPConfiguration, privatelinks.NATIPConfiguration(natIPConfiguration)) + } + privateLinksSpecs = append(privateLinksSpecs, &privateLinkSpec) + } + + return privateLinksSpecs +} + +func (s *ClusterScope) getLastAppliedSecurityRules(nsgName string) map[string]interface{} { // Retrieve the last applied security rules for all NSGs. lastAppliedSecurityRulesAll, err := s.AnnotationJSON(azure.SecurityRuleLastAppliedAnnotation) if err != nil { diff --git a/azure/scope/cluster_test.go b/azure/scope/cluster_test.go index a22393a532c..f6338474033 100644 --- a/azure/scope/cluster_test.go +++ b/azure/scope/cluster_test.go @@ -44,6 +44,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure/services/loadbalancers" "sigs.k8s.io/cluster-api-provider-azure/azure/services/natgateways" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/privatelinks" "sigs.k8s.io/cluster-api-provider-azure/azure/services/publicips" "sigs.k8s.io/cluster-api-provider-azure/azure/services/routetables" "sigs.k8s.io/cluster-api-provider-azure/azure/services/securitygroups" @@ -3962,6 +3963,207 @@ func TestSetFailureDomain(t *testing.T) { } } +func TestPrivateLinks(t *testing.T) { + fakeSubscriptionID := "123" + fakeResourceGroup := "my-rg" + fakeLocation := "westeurope" + fakeClusterName := "private-cluster" + fakeClusterNamespace := "hello" + fakeVNetResourceGroup := fakeResourceGroup + fakeVNetName := fmt.Sprintf("%s-vnet", fakeClusterName) + fakeSubnetName := "node-subnet" + fakeAPILBName := fmt.Sprintf("%s-apiserver-lb", fakeClusterName) + + tests := []struct { + name string + azureCluster infrav1.AzureCluster + expectedPrivateLinkSpecs []azure.ResourceSpecGetter + }{ + { + name: "AzureCluster with a private link", + azureCluster: infrav1.AzureCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: fakeClusterName, + Namespace: fakeClusterNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "Cluster", + Name: fakeClusterName, + }, + }, + }, + Spec: infrav1.AzureClusterSpec{ + ResourceGroup: fakeResourceGroup, + AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ + Location: fakeLocation, + SubscriptionID: fakeSubscriptionID, + IdentityRef: &corev1.ObjectReference{ + Kind: infrav1.AzureClusterIdentityKind, + }, + }, + NetworkSpec: infrav1.NetworkSpec{ + APIServerLB: &infrav1.LoadBalancerSpec{ + Name: fakeAPILBName, + FrontendIPs: []infrav1.FrontendIP{ + { + Name: fmt.Sprintf("%s-frontend", fakeAPILBName), + }, + }, + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + PrivateLinks: []infrav1.PrivateLink{ + { + Name: fmt.Sprintf("%s-privatelink", fakeAPILBName), + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: string(infrav1.NATIPAllocationMethodDynamic), + Subnet: fakeSubnetName, + }, + }, + LBFrontendIPConfigNames: []string{ + fmt.Sprintf("%s-frontend", fakeAPILBName), + }, + AllowedSubscriptions: []*string{ + &fakeSubscriptionID, + }, + AutoApprovedSubscriptions: []*string{ + &fakeSubscriptionID, + }, + }, + }, + }, + Vnet: infrav1.VnetSpec{ + ResourceGroup: fakeVNetResourceGroup, + Name: fakeVNetName, + }, + Subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: fakeSubnetName, + }, + }, + }, + }, + }, + }, + expectedPrivateLinkSpecs: []azure.ResourceSpecGetter{ + &privatelinks.PrivateLinkSpec{ + Name: fmt.Sprintf("%s-privatelink", fakeAPILBName), + ResourceGroup: fakeResourceGroup, + SubscriptionID: fakeSubscriptionID, + Location: fakeLocation, + VNetResourceGroup: fakeVNetResourceGroup, + VNet: fakeVNetName, + LoadBalancerName: fakeAPILBName, + LBFrontendIPConfigNames: []string{ + fmt.Sprintf("%s-frontend", fakeAPILBName), + }, + NATIPConfiguration: []privatelinks.NATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: fakeSubnetName, + }, + }, + AllowedSubscriptions: []*string{ + &fakeSubscriptionID, + }, + AutoApprovedSubscriptions: []*string{ + &fakeSubscriptionID, + }, + ClusterName: fakeClusterName, + AdditionalTags: map[string]string{}, + }, + }, + }, + { + name: "AzureCluster without private links", + azureCluster: infrav1.AzureCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: fakeClusterName, + Namespace: fakeClusterNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "Cluster", + Name: fakeClusterName, + }, + }, + }, + Spec: infrav1.AzureClusterSpec{ + ResourceGroup: fakeResourceGroup, + AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ + Location: fakeLocation, + SubscriptionID: fakeSubscriptionID, + IdentityRef: &corev1.ObjectReference{ + Kind: infrav1.AzureClusterIdentityKind, + }, + }, + NetworkSpec: infrav1.NetworkSpec{ + APIServerLB: &infrav1.LoadBalancerSpec{ + Name: fakeAPILBName, + FrontendIPs: []infrav1.FrontendIP{ + { + Name: fmt.Sprintf("%s-frontend", fakeAPILBName), + }, + }, + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + PrivateLinks: []infrav1.PrivateLink{}, + }, + }, + }, + }, + expectedPrivateLinkSpecs: []azure.ResourceSpecGetter{}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + _ = clusterv1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: fakeClusterName, + Namespace: fakeClusterNamespace, + }, + } + + fakeIdentity := &infrav1.AzureClusterIdentity{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: fakeClusterNamespace, + }, + Spec: infrav1.AzureClusterIdentitySpec{ + Type: infrav1.ServicePrincipal, + ClientID: fakeClientID, + TenantID: fakeTenantID, + }, + } + fakeSecret := &corev1.Secret{Data: map[string][]byte{"clientSecret": []byte("fooSecret")}} + + initObjects := []runtime.Object{cluster, &tc.azureCluster, fakeIdentity, fakeSecret} + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(initObjects...).Build() + + clusterScope := ClusterScope{ + AzureClients: AzureClients{ + subscriptionID: "123", + }, + Cluster: cluster, + AzureCluster: &tc.azureCluster, + Client: fakeClient, + } + got := clusterScope.PrivateLinkSpecs() + g.Expect(tc.expectedPrivateLinkSpecs).To(Equal(got)) + }) + } +} + func TestGroupSpecs(t *testing.T) { cases := []struct { name string diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 69df8abcabe..8bf3c956b6f 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -861,6 +861,11 @@ func (s *ManagedControlPlaneScope) SetLongRunningOperationState(future *infrav1. futures.Set(s.ControlPlane, future) } +// GetLongRunningOperationStates will get the specified futures on the AzureCluster status. +func (s *ManagedControlPlaneScope) GetLongRunningOperationStates(service, futureType string) infrav1.Futures { + return futures.GetByServiceAndType(s.ControlPlane, service, futureType) +} + // GetLongRunningOperationState will get the future on the AzureManagedControlPlane status. func (s *ManagedControlPlaneScope) GetLongRunningOperationState(name, service, futureType string) *infrav1.Future { return futures.Get(s.ControlPlane, name, service, futureType) diff --git a/azure/services/privateendpoints/privateendpoints.go b/azure/services/privateendpoints/privateendpoints.go index b3c09809381..67633ea5a7f 100644 --- a/azure/services/privateendpoints/privateendpoints.go +++ b/azure/services/privateendpoints/privateendpoints.go @@ -17,11 +17,15 @@ limitations under the License. package privateendpoints import ( + "context" + asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" + "sigs.k8s.io/controller-runtime/pkg/client" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/aso" + "sigs.k8s.io/cluster-api-provider-azure/util/slice" ) // ServiceName is the name of this service. @@ -36,7 +40,14 @@ type PrivateEndpointScope interface { // New creates a new service. func New(scope PrivateEndpointScope) *aso.Service[*asonetworkv1.PrivateEndpoint, PrivateEndpointScope] { svc := aso.NewService[*asonetworkv1.PrivateEndpoint, PrivateEndpointScope](ServiceName, scope) + svc.ListFunc = list svc.ConditionType = infrav1.PrivateEndpointsReadyCondition svc.Specs = scope.PrivateEndpointSpecs() return svc } + +func list(ctx context.Context, client client.Client, opts ...client.ListOption) ([]*asonetworkv1.PrivateEndpoint, error) { + list := &asonetworkv1.PrivateEndpointList{} + err := client.List(ctx, list, opts...) + return slice.ToPtrs(list.Items), err +} diff --git a/azure/services/privatelinks/client.go b/azure/services/privatelinks/client.go new file mode 100644 index 00000000000..7c8b9e48d30 --- /dev/null +++ b/azure/services/privatelinks/client.go @@ -0,0 +1,118 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package privatelinks + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + "github.com/pkg/errors" + + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +// azureClient contains the Azure go-sdk Client. +type azureClient struct { + privateLinks *armnetwork.PrivateLinkServicesClient +} + +// newClient creates a new route tables client from an authorizer. +func newClient(auth azure.Authorizer) (*azureClient, error) { + opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + if err != nil { + return nil, errors.Wrap(err, "failed to create routetables client options") + } + factory, err := armnetwork.NewClientFactory(auth.SubscriptionID(), auth.Token(), opts) + if err != nil { + return nil, errors.Wrap(err, "failed to create armnetwork client factory") + } + return &azureClient{factory.NewPrivateLinkServicesClient()}, nil +} + +// Get returns the specified private link. +func (ac *azureClient) Get(ctx context.Context, spec azure.ResourceSpecGetter) (result any, err error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "privatelinks.azureClient.Get") + defer done() + + resp, err := ac.privateLinks.Get(ctx, spec.ResourceGroupName(), spec.ResourceName(), nil) + if err != nil { + return nil, err + } + return resp.PrivateLinkService, nil +} + +// CreateOrUpdateAsync creates or updates a private link asynchronously. +// It sends a PUT request to Azure and if accepted without error, the func will return a Future which can be used to track the ongoing +// progress of the operation. +func (ac *azureClient) CreateOrUpdateAsync(ctx context.Context, spec azure.ResourceSpecGetter, resumeToken string, parameters any) (result any, poller *runtime.Poller[armnetwork.PrivateLinkServicesClientCreateOrUpdateResponse], err error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "privatelinks.azureClient.CreateOrUpdateAsync") + defer done() + + privateLink, ok := parameters.(armnetwork.PrivateLinkService) + if !ok { + return nil, nil, errors.Errorf("%T is not a network.PrivateLinkService", parameters) + } + + opts := &armnetwork.PrivateLinkServicesClientBeginCreateOrUpdateOptions{ResumeToken: resumeToken} + poller, err = ac.privateLinks.BeginCreateOrUpdate(ctx, spec.ResourceGroupName(), spec.ResourceName(), privateLink, opts) + if err != nil { + return nil, nil, err + } + + ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureCallTimeout) + defer cancel() + + pollOpts := &runtime.PollUntilDoneOptions{Frequency: async.DefaultPollerFrequency} + resp, err := poller.PollUntilDone(ctx, pollOpts) + if err != nil { + // if an error occurs, return the poller. + // this means the long-running operation didn't finish in the specified timeout. + return nil, poller, err + } + + // if the operation completed, return a nil poller + return resp.PrivateLinkService, nil, err +} + +// DeleteAsync deletes a private link asynchronously. DeleteAsync sends a DELETE +// request to Azure and if accepted without error, the func will return a Future which can be used to track the ongoing +// progress of the operation. +func (ac *azureClient) DeleteAsync(ctx context.Context, spec azure.ResourceSpecGetter, resumeToken string) (poller *runtime.Poller[armnetwork.PrivateLinkServicesClientDeleteResponse], err error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "privatelinks.azureClient.DeleteAsync") + defer done() + + opts := &armnetwork.PrivateLinkServicesClientBeginDeleteOptions{ResumeToken: resumeToken} + poller, err = ac.privateLinks.BeginDelete(ctx, spec.ResourceGroupName(), spec.ResourceName(), opts) + if err != nil { + return nil, err + } + + pollOpts := &runtime.PollUntilDoneOptions{Frequency: async.DefaultPollerFrequency} + _, err = poller.PollUntilDone(ctx, pollOpts) + if err != nil { + // if an error occurs, return the poller. + // this means the long-running operation didn't finish in the specified timeout. + return poller, err + } + + // if the operation completed, return a nil poller. + return nil, err +} diff --git a/azure/services/privatelinks/mock_privatelinks/doc.go b/azure/services/privatelinks/mock_privatelinks/doc.go new file mode 100644 index 00000000000..def472d79ed --- /dev/null +++ b/azure/services/privatelinks/mock_privatelinks/doc.go @@ -0,0 +1,21 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Run go generate to regenerate this mock. +// +//go:generate ../../../../hack/tools/bin/mockgen -destination privatelinks_mock.go -package mock_privatelinks -source ../privatelinks.go PrivateLinkScope +//go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt privatelinks_mock.go > _privatelinks_mock.go && mv _privatelinks_mock.go privatelinks_mock.go" +package mock_privatelinks diff --git a/azure/services/privatelinks/mock_privatelinks/privatelinks_mock.go b/azure/services/privatelinks/mock_privatelinks/privatelinks_mock.go new file mode 100644 index 00000000000..0304ddd02fa --- /dev/null +++ b/azure/services/privatelinks/mock_privatelinks/privatelinks_mock.go @@ -0,0 +1,709 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by MockGen. DO NOT EDIT. +// Source: ../privatelinks.go +// +// Generated by this command: +// +// mockgen -destination privatelinks_mock.go -package mock_privatelinks -source ../privatelinks.go PrivateLinkScope +// + +// Package mock_privatelinks is a generated GoMock package. +package mock_privatelinks + +import ( + reflect "reflect" + time "time" + + azcore "github.com/Azure/azure-sdk-for-go/sdk/azcore" + gomock "go.uber.org/mock/gomock" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1beta1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + azure "sigs.k8s.io/cluster-api-provider-azure/azure" + v1beta10 "sigs.k8s.io/cluster-api/api/core/v1beta1" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MockPrivateLinkScope is a mock of PrivateLinkScope interface. +type MockPrivateLinkScope struct { + ctrl *gomock.Controller + recorder *MockPrivateLinkScopeMockRecorder + isgomock struct{} +} + +// MockPrivateLinkScopeMockRecorder is the mock recorder for MockPrivateLinkScope. +type MockPrivateLinkScopeMockRecorder struct { + mock *MockPrivateLinkScope +} + +// NewMockPrivateLinkScope creates a new mock instance. +func NewMockPrivateLinkScope(ctrl *gomock.Controller) *MockPrivateLinkScope { + mock := &MockPrivateLinkScope{ctrl: ctrl} + mock.recorder = &MockPrivateLinkScopeMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPrivateLinkScope) EXPECT() *MockPrivateLinkScopeMockRecorder { + return m.recorder +} + +// APIServerLB mocks base method. +func (m *MockPrivateLinkScope) APIServerLB() *v1beta1.LoadBalancerSpec { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLB") + ret0, _ := ret[0].(*v1beta1.LoadBalancerSpec) + return ret0 +} + +// APIServerLB indicates an expected call of APIServerLB. +func (mr *MockPrivateLinkScopeMockRecorder) APIServerLB() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLB", reflect.TypeOf((*MockPrivateLinkScope)(nil).APIServerLB)) +} + +// APIServerLBName mocks base method. +func (m *MockPrivateLinkScope) APIServerLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBName indicates an expected call of APIServerLBName. +func (mr *MockPrivateLinkScopeMockRecorder) APIServerLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockPrivateLinkScope)(nil).APIServerLBName)) +} + +// APIServerLBPoolName mocks base method. +func (m *MockPrivateLinkScope) APIServerLBPoolName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBPoolName") + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBPoolName indicates an expected call of APIServerLBPoolName. +func (mr *MockPrivateLinkScopeMockRecorder) APIServerLBPoolName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBPoolName", reflect.TypeOf((*MockPrivateLinkScope)(nil).APIServerLBPoolName)) +} + +// AdditionalTags mocks base method. +func (m *MockPrivateLinkScope) AdditionalTags() v1beta1.Tags { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdditionalTags") + ret0, _ := ret[0].(v1beta1.Tags) + return ret0 +} + +// AdditionalTags indicates an expected call of AdditionalTags. +func (mr *MockPrivateLinkScopeMockRecorder) AdditionalTags() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdditionalTags", reflect.TypeOf((*MockPrivateLinkScope)(nil).AdditionalTags)) +} + +// AvailabilitySetEnabled mocks base method. +func (m *MockPrivateLinkScope) AvailabilitySetEnabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AvailabilitySetEnabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// AvailabilitySetEnabled indicates an expected call of AvailabilitySetEnabled. +func (mr *MockPrivateLinkScopeMockRecorder) AvailabilitySetEnabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AvailabilitySetEnabled", reflect.TypeOf((*MockPrivateLinkScope)(nil).AvailabilitySetEnabled)) +} + +// BaseURI mocks base method. +func (m *MockPrivateLinkScope) BaseURI() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BaseURI") + ret0, _ := ret[0].(string) + return ret0 +} + +// BaseURI indicates an expected call of BaseURI. +func (mr *MockPrivateLinkScopeMockRecorder) BaseURI() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BaseURI", reflect.TypeOf((*MockPrivateLinkScope)(nil).BaseURI)) +} + +// ClientID mocks base method. +func (m *MockPrivateLinkScope) ClientID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClientID") + ret0, _ := ret[0].(string) + return ret0 +} + +// ClientID indicates an expected call of ClientID. +func (mr *MockPrivateLinkScopeMockRecorder) ClientID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientID", reflect.TypeOf((*MockPrivateLinkScope)(nil).ClientID)) +} + +// ClientSecret mocks base method. +func (m *MockPrivateLinkScope) ClientSecret() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClientSecret") + ret0, _ := ret[0].(string) + return ret0 +} + +// ClientSecret indicates an expected call of ClientSecret. +func (mr *MockPrivateLinkScopeMockRecorder) ClientSecret() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientSecret", reflect.TypeOf((*MockPrivateLinkScope)(nil).ClientSecret)) +} + +// CloudEnvironment mocks base method. +func (m *MockPrivateLinkScope) CloudEnvironment() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloudEnvironment") + ret0, _ := ret[0].(string) + return ret0 +} + +// CloudEnvironment indicates an expected call of CloudEnvironment. +func (mr *MockPrivateLinkScopeMockRecorder) CloudEnvironment() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloudEnvironment", reflect.TypeOf((*MockPrivateLinkScope)(nil).CloudEnvironment)) +} + +// CloudProviderConfigOverrides mocks base method. +func (m *MockPrivateLinkScope) CloudProviderConfigOverrides() *v1beta1.CloudProviderConfigOverrides { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloudProviderConfigOverrides") + ret0, _ := ret[0].(*v1beta1.CloudProviderConfigOverrides) + return ret0 +} + +// CloudProviderConfigOverrides indicates an expected call of CloudProviderConfigOverrides. +func (mr *MockPrivateLinkScopeMockRecorder) CloudProviderConfigOverrides() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloudProviderConfigOverrides", reflect.TypeOf((*MockPrivateLinkScope)(nil).CloudProviderConfigOverrides)) +} + +// ClusterName mocks base method. +func (m *MockPrivateLinkScope) ClusterName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClusterName") + ret0, _ := ret[0].(string) + return ret0 +} + +// ClusterName indicates an expected call of ClusterName. +func (mr *MockPrivateLinkScopeMockRecorder) ClusterName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterName", reflect.TypeOf((*MockPrivateLinkScope)(nil).ClusterName)) +} + +// ControlPlaneRouteTable mocks base method. +func (m *MockPrivateLinkScope) ControlPlaneRouteTable() v1beta1.RouteTable { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ControlPlaneRouteTable") + ret0, _ := ret[0].(v1beta1.RouteTable) + return ret0 +} + +// ControlPlaneRouteTable indicates an expected call of ControlPlaneRouteTable. +func (mr *MockPrivateLinkScopeMockRecorder) ControlPlaneRouteTable() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ControlPlaneRouteTable", reflect.TypeOf((*MockPrivateLinkScope)(nil).ControlPlaneRouteTable)) +} + +// ControlPlaneSubnet mocks base method. +func (m *MockPrivateLinkScope) ControlPlaneSubnet() v1beta1.SubnetSpec { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ControlPlaneSubnet") + ret0, _ := ret[0].(v1beta1.SubnetSpec) + return ret0 +} + +// ControlPlaneSubnet indicates an expected call of ControlPlaneSubnet. +func (mr *MockPrivateLinkScopeMockRecorder) ControlPlaneSubnet() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ControlPlaneSubnet", reflect.TypeOf((*MockPrivateLinkScope)(nil).ControlPlaneSubnet)) +} + +// DefaultedAzureCallTimeout mocks base method. +func (m *MockPrivateLinkScope) DefaultedAzureCallTimeout() time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DefaultedAzureCallTimeout") + ret0, _ := ret[0].(time.Duration) + return ret0 +} + +// DefaultedAzureCallTimeout indicates an expected call of DefaultedAzureCallTimeout. +func (mr *MockPrivateLinkScopeMockRecorder) DefaultedAzureCallTimeout() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DefaultedAzureCallTimeout", reflect.TypeOf((*MockPrivateLinkScope)(nil).DefaultedAzureCallTimeout)) +} + +// DefaultedAzureServiceReconcileTimeout mocks base method. +func (m *MockPrivateLinkScope) DefaultedAzureServiceReconcileTimeout() time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DefaultedAzureServiceReconcileTimeout") + ret0, _ := ret[0].(time.Duration) + return ret0 +} + +// DefaultedAzureServiceReconcileTimeout indicates an expected call of DefaultedAzureServiceReconcileTimeout. +func (mr *MockPrivateLinkScopeMockRecorder) DefaultedAzureServiceReconcileTimeout() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DefaultedAzureServiceReconcileTimeout", reflect.TypeOf((*MockPrivateLinkScope)(nil).DefaultedAzureServiceReconcileTimeout)) +} + +// DefaultedReconcilerRequeue mocks base method. +func (m *MockPrivateLinkScope) DefaultedReconcilerRequeue() time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DefaultedReconcilerRequeue") + ret0, _ := ret[0].(time.Duration) + return ret0 +} + +// DefaultedReconcilerRequeue indicates an expected call of DefaultedReconcilerRequeue. +func (mr *MockPrivateLinkScopeMockRecorder) DefaultedReconcilerRequeue() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DefaultedReconcilerRequeue", reflect.TypeOf((*MockPrivateLinkScope)(nil).DefaultedReconcilerRequeue)) +} + +// DeleteLongRunningOperationState mocks base method. +func (m *MockPrivateLinkScope) DeleteLongRunningOperationState(arg0, arg1, arg2 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "DeleteLongRunningOperationState", arg0, arg1, arg2) +} + +// DeleteLongRunningOperationState indicates an expected call of DeleteLongRunningOperationState. +func (mr *MockPrivateLinkScopeMockRecorder) DeleteLongRunningOperationState(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLongRunningOperationState", reflect.TypeOf((*MockPrivateLinkScope)(nil).DeleteLongRunningOperationState), arg0, arg1, arg2) +} + +// ExtendedLocation mocks base method. +func (m *MockPrivateLinkScope) ExtendedLocation() *v1beta1.ExtendedLocationSpec { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtendedLocation") + ret0, _ := ret[0].(*v1beta1.ExtendedLocationSpec) + return ret0 +} + +// ExtendedLocation indicates an expected call of ExtendedLocation. +func (mr *MockPrivateLinkScopeMockRecorder) ExtendedLocation() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtendedLocation", reflect.TypeOf((*MockPrivateLinkScope)(nil).ExtendedLocation)) +} + +// ExtendedLocationName mocks base method. +func (m *MockPrivateLinkScope) ExtendedLocationName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtendedLocationName") + ret0, _ := ret[0].(string) + return ret0 +} + +// ExtendedLocationName indicates an expected call of ExtendedLocationName. +func (mr *MockPrivateLinkScopeMockRecorder) ExtendedLocationName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtendedLocationName", reflect.TypeOf((*MockPrivateLinkScope)(nil).ExtendedLocationName)) +} + +// ExtendedLocationType mocks base method. +func (m *MockPrivateLinkScope) ExtendedLocationType() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtendedLocationType") + ret0, _ := ret[0].(string) + return ret0 +} + +// ExtendedLocationType indicates an expected call of ExtendedLocationType. +func (mr *MockPrivateLinkScopeMockRecorder) ExtendedLocationType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtendedLocationType", reflect.TypeOf((*MockPrivateLinkScope)(nil).ExtendedLocationType)) +} + +// FailureDomains mocks base method. +func (m *MockPrivateLinkScope) FailureDomains() []*string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FailureDomains") + ret0, _ := ret[0].([]*string) + return ret0 +} + +// FailureDomains indicates an expected call of FailureDomains. +func (mr *MockPrivateLinkScopeMockRecorder) FailureDomains() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FailureDomains", reflect.TypeOf((*MockPrivateLinkScope)(nil).FailureDomains)) +} + +// GetClient mocks base method. +func (m *MockPrivateLinkScope) GetClient() client.Client { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClient") + ret0, _ := ret[0].(client.Client) + return ret0 +} + +// GetClient indicates an expected call of GetClient. +func (mr *MockPrivateLinkScopeMockRecorder) GetClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClient", reflect.TypeOf((*MockPrivateLinkScope)(nil).GetClient)) +} + +// GetDeletionTimestamp mocks base method. +func (m *MockPrivateLinkScope) GetDeletionTimestamp() *v1.Time { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDeletionTimestamp") + ret0, _ := ret[0].(*v1.Time) + return ret0 +} + +// GetDeletionTimestamp indicates an expected call of GetDeletionTimestamp. +func (mr *MockPrivateLinkScopeMockRecorder) GetDeletionTimestamp() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeletionTimestamp", reflect.TypeOf((*MockPrivateLinkScope)(nil).GetDeletionTimestamp)) +} + +// GetLongRunningOperationState mocks base method. +func (m *MockPrivateLinkScope) GetLongRunningOperationState(arg0, arg1, arg2 string) *v1beta1.Future { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLongRunningOperationState", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1beta1.Future) + return ret0 +} + +// GetLongRunningOperationState indicates an expected call of GetLongRunningOperationState. +func (mr *MockPrivateLinkScopeMockRecorder) GetLongRunningOperationState(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLongRunningOperationState", reflect.TypeOf((*MockPrivateLinkScope)(nil).GetLongRunningOperationState), arg0, arg1, arg2) +} + +// GetPrivateDNSZoneName mocks base method. +func (m *MockPrivateLinkScope) GetPrivateDNSZoneName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPrivateDNSZoneName") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetPrivateDNSZoneName indicates an expected call of GetPrivateDNSZoneName. +func (mr *MockPrivateLinkScopeMockRecorder) GetPrivateDNSZoneName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrivateDNSZoneName", reflect.TypeOf((*MockPrivateLinkScope)(nil).GetPrivateDNSZoneName)) +} + +// HashKey mocks base method. +func (m *MockPrivateLinkScope) HashKey() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HashKey") + ret0, _ := ret[0].(string) + return ret0 +} + +// HashKey indicates an expected call of HashKey. +func (mr *MockPrivateLinkScopeMockRecorder) HashKey() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HashKey", reflect.TypeOf((*MockPrivateLinkScope)(nil).HashKey)) +} + +// IsAPIServerPrivate mocks base method. +func (m *MockPrivateLinkScope) IsAPIServerPrivate() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAPIServerPrivate") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAPIServerPrivate indicates an expected call of IsAPIServerPrivate. +func (mr *MockPrivateLinkScopeMockRecorder) IsAPIServerPrivate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAPIServerPrivate", reflect.TypeOf((*MockPrivateLinkScope)(nil).IsAPIServerPrivate)) +} + +// IsIPv6Enabled mocks base method. +func (m *MockPrivateLinkScope) IsIPv6Enabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsIPv6Enabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsIPv6Enabled indicates an expected call of IsIPv6Enabled. +func (mr *MockPrivateLinkScopeMockRecorder) IsIPv6Enabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsIPv6Enabled", reflect.TypeOf((*MockPrivateLinkScope)(nil).IsIPv6Enabled)) +} + +// IsVnetManaged mocks base method. +func (m *MockPrivateLinkScope) IsVnetManaged() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsVnetManaged") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsVnetManaged indicates an expected call of IsVnetManaged. +func (mr *MockPrivateLinkScopeMockRecorder) IsVnetManaged() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsVnetManaged", reflect.TypeOf((*MockPrivateLinkScope)(nil).IsVnetManaged)) +} + +// Location mocks base method. +func (m *MockPrivateLinkScope) Location() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Location") + ret0, _ := ret[0].(string) + return ret0 +} + +// Location indicates an expected call of Location. +func (mr *MockPrivateLinkScopeMockRecorder) Location() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Location", reflect.TypeOf((*MockPrivateLinkScope)(nil).Location)) +} + +// NodeResourceGroup mocks base method. +func (m *MockPrivateLinkScope) NodeResourceGroup() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeResourceGroup") + ret0, _ := ret[0].(string) + return ret0 +} + +// NodeResourceGroup indicates an expected call of NodeResourceGroup. +func (mr *MockPrivateLinkScopeMockRecorder) NodeResourceGroup() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeResourceGroup", reflect.TypeOf((*MockPrivateLinkScope)(nil).NodeResourceGroup)) +} + +// NodeSubnets mocks base method. +func (m *MockPrivateLinkScope) NodeSubnets() []v1beta1.SubnetSpec { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeSubnets") + ret0, _ := ret[0].([]v1beta1.SubnetSpec) + return ret0 +} + +// NodeSubnets indicates an expected call of NodeSubnets. +func (mr *MockPrivateLinkScopeMockRecorder) NodeSubnets() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeSubnets", reflect.TypeOf((*MockPrivateLinkScope)(nil).NodeSubnets)) +} + +// OutboundLBName mocks base method. +func (m *MockPrivateLinkScope) OutboundLBName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundLBName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundLBName indicates an expected call of OutboundLBName. +func (mr *MockPrivateLinkScopeMockRecorder) OutboundLBName(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundLBName", reflect.TypeOf((*MockPrivateLinkScope)(nil).OutboundLBName), arg0) +} + +// OutboundPoolName mocks base method. +func (m *MockPrivateLinkScope) OutboundPoolName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundPoolName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundPoolName indicates an expected call of OutboundPoolName. +func (mr *MockPrivateLinkScopeMockRecorder) OutboundPoolName(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundPoolName", reflect.TypeOf((*MockPrivateLinkScope)(nil).OutboundPoolName), arg0) +} + +// PrivateLinkSpecs mocks base method. +func (m *MockPrivateLinkScope) PrivateLinkSpecs() []azure.ResourceSpecGetter { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PrivateLinkSpecs") + ret0, _ := ret[0].([]azure.ResourceSpecGetter) + return ret0 +} + +// PrivateLinkSpecs indicates an expected call of PrivateLinkSpecs. +func (mr *MockPrivateLinkScopeMockRecorder) PrivateLinkSpecs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrivateLinkSpecs", reflect.TypeOf((*MockPrivateLinkScope)(nil).PrivateLinkSpecs)) +} + +// ResourceGroup mocks base method. +func (m *MockPrivateLinkScope) ResourceGroup() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResourceGroup") + ret0, _ := ret[0].(string) + return ret0 +} + +// ResourceGroup indicates an expected call of ResourceGroup. +func (mr *MockPrivateLinkScopeMockRecorder) ResourceGroup() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourceGroup", reflect.TypeOf((*MockPrivateLinkScope)(nil).ResourceGroup)) +} + +// SetLongRunningOperationState mocks base method. +func (m *MockPrivateLinkScope) SetLongRunningOperationState(arg0 *v1beta1.Future) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetLongRunningOperationState", arg0) +} + +// SetLongRunningOperationState indicates an expected call of SetLongRunningOperationState. +func (mr *MockPrivateLinkScopeMockRecorder) SetLongRunningOperationState(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLongRunningOperationState", reflect.TypeOf((*MockPrivateLinkScope)(nil).SetLongRunningOperationState), arg0) +} + +// SetSubnet mocks base method. +func (m *MockPrivateLinkScope) SetSubnet(arg0 v1beta1.SubnetSpec) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetSubnet", arg0) +} + +// SetSubnet indicates an expected call of SetSubnet. +func (mr *MockPrivateLinkScopeMockRecorder) SetSubnet(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetSubnet", reflect.TypeOf((*MockPrivateLinkScope)(nil).SetSubnet), arg0) +} + +// Subnet mocks base method. +func (m *MockPrivateLinkScope) Subnet(arg0 string) v1beta1.SubnetSpec { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Subnet", arg0) + ret0, _ := ret[0].(v1beta1.SubnetSpec) + return ret0 +} + +// Subnet indicates an expected call of Subnet. +func (mr *MockPrivateLinkScopeMockRecorder) Subnet(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Subnet", reflect.TypeOf((*MockPrivateLinkScope)(nil).Subnet), arg0) +} + +// Subnets mocks base method. +func (m *MockPrivateLinkScope) Subnets() v1beta1.Subnets { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Subnets") + ret0, _ := ret[0].(v1beta1.Subnets) + return ret0 +} + +// Subnets indicates an expected call of Subnets. +func (mr *MockPrivateLinkScopeMockRecorder) Subnets() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Subnets", reflect.TypeOf((*MockPrivateLinkScope)(nil).Subnets)) +} + +// SubscriptionID mocks base method. +func (m *MockPrivateLinkScope) SubscriptionID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SubscriptionID") + ret0, _ := ret[0].(string) + return ret0 +} + +// SubscriptionID indicates an expected call of SubscriptionID. +func (mr *MockPrivateLinkScopeMockRecorder) SubscriptionID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubscriptionID", reflect.TypeOf((*MockPrivateLinkScope)(nil).SubscriptionID)) +} + +// TenantID mocks base method. +func (m *MockPrivateLinkScope) TenantID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TenantID") + ret0, _ := ret[0].(string) + return ret0 +} + +// TenantID indicates an expected call of TenantID. +func (mr *MockPrivateLinkScopeMockRecorder) TenantID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TenantID", reflect.TypeOf((*MockPrivateLinkScope)(nil).TenantID)) +} + +// Token mocks base method. +func (m *MockPrivateLinkScope) Token() azcore.TokenCredential { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Token") + ret0, _ := ret[0].(azcore.TokenCredential) + return ret0 +} + +// Token indicates an expected call of Token. +func (mr *MockPrivateLinkScopeMockRecorder) Token() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Token", reflect.TypeOf((*MockPrivateLinkScope)(nil).Token)) +} + +// UpdateDeleteStatus mocks base method. +func (m *MockPrivateLinkScope) UpdateDeleteStatus(arg0 v1beta10.ConditionType, arg1 string, arg2 error) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "UpdateDeleteStatus", arg0, arg1, arg2) +} + +// UpdateDeleteStatus indicates an expected call of UpdateDeleteStatus. +func (mr *MockPrivateLinkScopeMockRecorder) UpdateDeleteStatus(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeleteStatus", reflect.TypeOf((*MockPrivateLinkScope)(nil).UpdateDeleteStatus), arg0, arg1, arg2) +} + +// UpdatePatchStatus mocks base method. +func (m *MockPrivateLinkScope) UpdatePatchStatus(arg0 v1beta10.ConditionType, arg1 string, arg2 error) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "UpdatePatchStatus", arg0, arg1, arg2) +} + +// UpdatePatchStatus indicates an expected call of UpdatePatchStatus. +func (mr *MockPrivateLinkScopeMockRecorder) UpdatePatchStatus(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePatchStatus", reflect.TypeOf((*MockPrivateLinkScope)(nil).UpdatePatchStatus), arg0, arg1, arg2) +} + +// UpdatePutStatus mocks base method. +func (m *MockPrivateLinkScope) UpdatePutStatus(arg0 v1beta10.ConditionType, arg1 string, arg2 error) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "UpdatePutStatus", arg0, arg1, arg2) +} + +// UpdatePutStatus indicates an expected call of UpdatePutStatus. +func (mr *MockPrivateLinkScopeMockRecorder) UpdatePutStatus(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePutStatus", reflect.TypeOf((*MockPrivateLinkScope)(nil).UpdatePutStatus), arg0, arg1, arg2) +} + +// Vnet mocks base method. +func (m *MockPrivateLinkScope) Vnet() *v1beta1.VnetSpec { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Vnet") + ret0, _ := ret[0].(*v1beta1.VnetSpec) + return ret0 +} + +// Vnet indicates an expected call of Vnet. +func (mr *MockPrivateLinkScopeMockRecorder) Vnet() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Vnet", reflect.TypeOf((*MockPrivateLinkScope)(nil).Vnet)) +} diff --git a/azure/services/privatelinks/privatelinks.go b/azure/services/privatelinks/privatelinks.go new file mode 100644 index 00000000000..b3786d6175a --- /dev/null +++ b/azure/services/privatelinks/privatelinks.go @@ -0,0 +1,120 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package privatelinks + +import ( + "context" + + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +// ServiceName is the name of this service. +const ServiceName = "privatelinks" + +// PrivateLinkScope defines the scope interface for a private link. +type PrivateLinkScope interface { + azure.ClusterScoper + azure.AsyncStatusUpdater + PrivateLinkSpecs() []azure.ResourceSpecGetter +} + +// Service provides operations on Azure resources. +type Service struct { + Scope PrivateLinkScope + async.Reconciler +} + +// New creates a new service. +func New(scope PrivateLinkScope) (*Service, error) { + client, err := newClient(scope) + if err != nil { + return nil, err + } + return &Service{ + Scope: scope, + Reconciler: async.New(scope, client, client), + }, nil +} + +// Name returns the service name. +func (s *Service) Name() string { + return ServiceName +} + +// Reconcile idempotently creates or updates a private link. +func (s *Service) Reconcile(ctx context.Context) error { + ctx, _, done := tele.StartSpanWithLogger(ctx, "privatelinks.Service.Reconcile") + defer done() + + ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout) + defer cancel() + + specs := s.Scope.PrivateLinkSpecs() + if len(specs) == 0 { + return nil + } + + var resultingErr error + for _, privateLinkSpec := range specs { + _, err := s.CreateOrUpdateResource(ctx, privateLinkSpec, ServiceName) + if err != nil { + if !azure.IsOperationNotDoneError(err) || resultingErr == nil { + resultingErr = err + } + } + } + + s.Scope.UpdatePutStatus(infrav1.PrivateLinksReadyCondition, ServiceName, resultingErr) + return resultingErr +} + +// Delete reconciles the private link deletion. +func (s *Service) Delete(ctx context.Context) error { + ctx, _, done := tele.StartSpanWithLogger(ctx, "privatelinks.Service.Delete") + defer done() + + ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout) + defer cancel() + + specs := s.Scope.PrivateLinkSpecs() + if len(specs) == 0 { + return nil + } + + // We go through the list of PrivateLinkSpecs to delete each one, independently of the resultingErr of the previous one. + // If multiple errors occur, we return the most pressing one. + // Order of precedence (highest -> lowest) is: error that is not an operationNotDoneError (i.e. error deleting) -> operationNotDoneError (i.e. deleting in progress) -> no error (i.e. deleted) + var resultingErr error + for _, privateLinkSpec := range specs { + if err := s.DeleteResource(ctx, privateLinkSpec, ServiceName); err != nil { + if !azure.IsOperationNotDoneError(err) || resultingErr == nil { + resultingErr = err + } + } + } + s.Scope.UpdateDeleteStatus(infrav1.PrivateLinksReadyCondition, ServiceName, resultingErr) + return resultingErr +} + +// IsManaged returns always returns true as CAPZ does not support BYO private links. +func (s *Service) IsManaged(_ context.Context) (bool, error) { + return true, nil +} diff --git a/azure/services/privatelinks/privatelinks_test.go b/azure/services/privatelinks/privatelinks_test.go new file mode 100644 index 00000000000..5403bf854b7 --- /dev/null +++ b/azure/services/privatelinks/privatelinks_test.go @@ -0,0 +1,300 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package privatelinks + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + "k8s.io/utils/ptr" + + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/async/mock_async" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/privatelinks/mock_privatelinks" + gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock" +) + +const ( + fakePrivateLinkValidSpec1Name = fakePrivateLinkName + "-valid-1" + fakePrivateLinkValidSpec2Name = fakePrivateLinkName + "-valid-2" +) + +var ( + // fakePrivateLinkValidSpec1 is valid private link spec with: + // - 1 allowed subscription, + // - 1 auto-approved subscription, + // - disabled proxy protocol and + // - additional tag "hello:capz". + fakePrivateLinkValidSpec1 = PrivateLinkSpec{ + Name: fakePrivateLinkValidSpec1Name, + ResourceGroup: fakeClusterName, + SubscriptionID: fakeSubscriptionID1, + Location: fakeRegion, + VNetResourceGroup: fakeVNetResourceGroup, + VNet: fakeVNetName, + NATIPConfiguration: []NATIPConfiguration{ + { + AllocationMethod: string(armnetwork.IPAllocationMethodDynamic), + Subnet: fakeSubnetName, + }, + }, + LoadBalancerName: fakeLbName, + LBFrontendIPConfigNames: []string{ + fakeLbIPConfigName1, + }, + AllowedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + AutoApprovedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + EnableProxyProtocol: ptr.To(false), + ClusterName: fakeClusterName, + AdditionalTags: map[string]string{ + "hello": "capz", + }, + } + + // fakePrivateLinkValidSpec2 is valid private link spec with: + // - 2 allowed subscriptions, + // - 2 auto-approved subscription, + // - enabled proxy protocol and + // - additional tag "hello:capz". + fakePrivateLinkValidSpec2 = PrivateLinkSpec{ + Name: fakePrivateLinkValidSpec2Name, + ResourceGroup: fakeClusterName, + SubscriptionID: fakeSubscriptionID1, + Location: fakeRegion, + VNetResourceGroup: fakeVNetResourceGroup, + VNet: fakeVNetName, + NATIPConfiguration: []NATIPConfiguration{ + { + AllocationMethod: string(armnetwork.IPAllocationMethodDynamic), + Subnet: fakeSubnetName, + }, + }, + LoadBalancerName: fakeLbName, + LBFrontendIPConfigNames: []string{ + fakeLbIPConfigName1, + }, + AllowedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + }, + AutoApprovedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + }, + EnableProxyProtocol: ptr.To(true), + ClusterName: fakeClusterName, + AdditionalTags: map[string]string{ + "hello": "capz", + }, + } + + emptyPrivateLinkSpec = PrivateLinkSpec{} + + internalError = &azcore.ResponseError{ + RawResponse: &http.Response{ + Body: io.NopCloser(strings.NewReader("Internal Server Error")), + StatusCode: http.StatusInternalServerError, + }, + } + notDoneError = azure.NewOperationNotDoneError(&infrav1.Future{}) +) + +func TestReconcilePrivateLink(t *testing.T) { + testcases := []struct { + name string + expect func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) + expectedError string + }{ + { + name: "successfully create one private link", + expectedError: "", + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateLinkValidSpec1}) + r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec1, ServiceName).Return(&fakePrivateLinkValidSpec1, nil) + s.UpdatePutStatus(infrav1.PrivateLinksReadyCondition, ServiceName, nil) + }, + }, + { + name: "successfully create multiple private links", + expectedError: "", + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateLinkValidSpec1, &fakePrivateLinkValidSpec2}) + r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec1, ServiceName).Return(&fakePrivateLinkValidSpec1, nil) + r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec2, ServiceName).Return(&fakePrivateLinkValidSpec2, nil) + s.UpdatePutStatus(infrav1.PrivateLinksReadyCondition, ServiceName, nil) + }, + }, + { + name: "error when creating a private link using an empty spec", + expectedError: internalError.Error(), + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&emptyPrivateLinkSpec}) + r.CreateOrUpdateResource(gomockinternal.AContext(), &emptyPrivateLinkSpec, ServiceName).Return(nil, internalError) + s.UpdatePutStatus(infrav1.PrivateLinksReadyCondition, ServiceName, internalError) + }, + }, + { + name: "not done error in creating is ignored", + expectedError: internalError.Error(), + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateLinkValidSpec1, &fakePrivateLinkValidSpec2}) + r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec1, ServiceName).Return(&fakePrivateLinkValidSpec1, internalError) + r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec2, ServiceName).Return(&fakePrivateLinkValidSpec2, notDoneError) + s.UpdatePutStatus(infrav1.PrivateLinksReadyCondition, ServiceName, internalError) + }, + }, + { + name: "not done error in creating remains", + expectedError: notDoneError.Error(), + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateLinkValidSpec1, &fakePrivateLinkValidSpec2}) + r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec1, ServiceName).Return(&fakePrivateLinkValidSpec1, notDoneError) + r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec2, ServiceName).Return(&fakePrivateLinkValidSpec2, nil) + s.UpdatePutStatus(infrav1.PrivateLinksReadyCondition, ServiceName, notDoneError) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + scopeMock := mock_privatelinks.NewMockPrivateLinkScope(mockCtrl) + asyncMock := mock_async.NewMockReconciler(mockCtrl) + + tc.expect(scopeMock.EXPECT(), asyncMock.EXPECT()) + + s := &Service{ + Scope: scopeMock, + Reconciler: asyncMock, + } + + err := s.Reconcile(context.TODO()) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + +func TestDeletePrivateLinks(t *testing.T) { + testcases := []struct { + name string + expectedError string + expect func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) + }{ + { + name: "delete a private link", + expectedError: "", + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateLinkValidSpec1}) + r.DeleteResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec1, ServiceName).Return(nil) + s.UpdateDeleteStatus(infrav1.PrivateLinksReadyCondition, ServiceName, nil) + }, + }, + { + name: "delete multiple private links", + expectedError: "", + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateLinkValidSpec1, &fakePrivateLinkValidSpec2}) + r.DeleteResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec1, ServiceName).Return(nil) + r.DeleteResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec2, ServiceName).Return(nil) + s.UpdateDeleteStatus(infrav1.PrivateLinksReadyCondition, ServiceName, nil) + }, + }, + { + name: "noop if no private link specs are found", + expectedError: "", + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{}) + }, + }, + { + name: "error when deleting a private link", + expectedError: internalError.Error(), + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateLinkValidSpec1, &fakePrivateLinkValidSpec2}) + r.DeleteResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec1, ServiceName).Return(nil) + r.DeleteResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec2, ServiceName).Return(internalError) + s.UpdateDeleteStatus(infrav1.PrivateLinksReadyCondition, ServiceName, internalError) + }, + }, + { + name: "not done error when deleting a private link is ignored", + expectedError: internalError.Error(), + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateLinkValidSpec1, &fakePrivateLinkValidSpec2}) + r.DeleteResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec1, ServiceName).Return(internalError) + r.DeleteResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec2, ServiceName).Return(notDoneError) + s.UpdateDeleteStatus(infrav1.PrivateLinksReadyCondition, ServiceName, internalError) + }, + }, + { + name: "not done error when deleting a private link remains", + expectedError: notDoneError.Error(), + expect: func(s *mock_privatelinks.MockPrivateLinkScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.PrivateLinkSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateLinkValidSpec1, &fakePrivateLinkValidSpec2}) + r.DeleteResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec1, ServiceName).Return(nil) + r.DeleteResource(gomockinternal.AContext(), &fakePrivateLinkValidSpec2, ServiceName).Return(notDoneError) + s.UpdateDeleteStatus(infrav1.PrivateLinksReadyCondition, ServiceName, notDoneError) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + scopeMock := mock_privatelinks.NewMockPrivateLinkScope(mockCtrl) + asyncMock := mock_async.NewMockReconciler(mockCtrl) + + tc.expect(scopeMock.EXPECT(), asyncMock.EXPECT()) + + s := &Service{ + Scope: scopeMock, + Reconciler: asyncMock, + } + + err := s.Delete(context.TODO()) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} diff --git a/azure/services/privatelinks/spec.go b/azure/services/privatelinks/spec.go new file mode 100644 index 00000000000..7566b105e32 --- /dev/null +++ b/azure/services/privatelinks/spec.go @@ -0,0 +1,243 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package privatelinks + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + "github.com/pkg/errors" + "k8s.io/utils/ptr" + + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/converters" +) + +// PrivateLinkSpec defines the specification for a private link service. +type PrivateLinkSpec struct { + Name string + ResourceGroup string + SubscriptionID string + Location string + VNetResourceGroup string + VNet string + NATIPConfiguration []NATIPConfiguration + LoadBalancerName string + LBFrontendIPConfigNames []string + AllowedSubscriptions []*string + AutoApprovedSubscriptions []*string + EnableProxyProtocol *bool + ClusterName string + AdditionalTags infrav1.Tags +} + +// NATIPConfiguration defines the NAT IP configuration for the private link service. +type NATIPConfiguration struct { + // AllocationMethod can be Static or Dynamic. + AllocationMethod string + + // Subnet from the VNet from which the IP is allocated. + Subnet string + + // PrivateIPAddress is the optional static private IP address from the specified Subnet. + PrivateIPAddress string +} + +// ResourceName returns the name of the private link. +func (s *PrivateLinkSpec) ResourceName() string { + return s.Name +} + +// ResourceGroupName returns the name of the resource group. +func (s *PrivateLinkSpec) ResourceGroupName() string { + return s.ResourceGroup +} + +// OwnerResourceName is a no-op for private link. +func (s *PrivateLinkSpec) OwnerResourceName() string { + return "" +} + +// Parameters returns the parameters for the private link. +func (s *PrivateLinkSpec) Parameters(_ context.Context, existing any) (params any, err error) { + if existing != nil { + // Private link already exist, so we have to check if it should be updated. + existingPrivateLink, ok := existing.(armnetwork.PrivateLinkService) + if !ok { + return nil, errors.Errorf("%T is not a armnetwork.PrivateLinkService", existing) + } + + privateLinkToCreate, err := s.constructParameters() + if err != nil { + return nil, err + } + + if isExistingUpToDate(existingPrivateLink, privateLinkToCreate) { + // Existing private link is up-to-date. + return nil, nil + } + + // Existing private link is outdated, we return new updated parameters. + return privateLinkToCreate, nil + } + + // Private link does not exist, so we create it here. + privateLinkToCreate, err := s.constructParameters() + if err != nil { + return nil, err + } + + return privateLinkToCreate, nil +} + +func (s *PrivateLinkSpec) constructParameters() (params armnetwork.PrivateLinkService, err error) { + if len(s.NATIPConfiguration) == 0 { + return armnetwork.PrivateLinkService{}, errors.Errorf("At least one private link NAT IP configuration must be specified") + } + if len(s.LBFrontendIPConfigNames) == 0 { + return armnetwork.PrivateLinkService{}, errors.Errorf("At least one load balancer front end name must be specified") + } + + // NAT IP configurations + ipConfigurations := make([]*armnetwork.PrivateLinkServiceIPConfiguration, 0, len(s.NATIPConfiguration)) + for i, natIPConfiguration := range s.NATIPConfiguration { + ipAllocationMethod := armnetwork.IPAllocationMethod(natIPConfiguration.AllocationMethod) + if ipAllocationMethod != armnetwork.IPAllocationMethodDynamic && ipAllocationMethod != armnetwork.IPAllocationMethodStatic { + return armnetwork.PrivateLinkService{}, errors.Errorf("%T is not a supported armnetwork.IPAllocationMethodStatic", natIPConfiguration.AllocationMethod) + } + var privateIPAddress *string + if ipAllocationMethod == armnetwork.IPAllocationMethodStatic { + if natIPConfiguration.PrivateIPAddress != "" { + privateIPAddress = ptr.To(natIPConfiguration.PrivateIPAddress) + } else { + return armnetwork.PrivateLinkService{}, errors.Errorf("Private link NAT IP configuration with static IP allocation must specify a private address") + } + } + ipConfiguration := armnetwork.PrivateLinkServiceIPConfiguration{ + Name: ptr.To(fmt.Sprintf("%s-natipconfig-%d", natIPConfiguration.Subnet, i+1)), + Properties: &armnetwork.PrivateLinkServiceIPConfigurationProperties{ + Subnet: &armnetwork.Subnet{ + ID: ptr.To(azure.SubnetID(s.SubscriptionID, s.VNetResourceGroup, s.VNet, natIPConfiguration.Subnet)), + }, + PrivateIPAllocationMethod: &ipAllocationMethod, + PrivateIPAddress: privateIPAddress, + }, + } + ipConfigurations = append(ipConfigurations, &ipConfiguration) + ipConfigurations[0].Properties.Primary = ptr.To(true) + } + + // Load balancer front-end IP configurations + frontendIPConfigurations := make([]*armnetwork.FrontendIPConfiguration, 0, len(s.LBFrontendIPConfigNames)) + for _, frontendIPConfigName := range s.LBFrontendIPConfigNames { + frontendIPConfig := armnetwork.FrontendIPConfiguration{ + ID: ptr.To(azure.FrontendIPConfigID(s.SubscriptionID, s.ResourceGroupName(), s.LoadBalancerName, frontendIPConfigName)), + } + frontendIPConfigurations = append(frontendIPConfigurations, &frontendIPConfig) + } + + privateLinkToCreate := armnetwork.PrivateLinkService{ + Name: ptr.To(s.Name), + Location: ptr.To(s.Location), + Properties: &armnetwork.PrivateLinkServiceProperties{ + IPConfigurations: ipConfigurations, + LoadBalancerFrontendIPConfigurations: frontendIPConfigurations, + EnableProxyProtocol: s.EnableProxyProtocol, + }, + Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ + ClusterName: s.ClusterName, + Lifecycle: infrav1.ResourceLifecycleOwned, + Name: ptr.To(s.Name), + Additional: s.AdditionalTags, + })), + } + + if len(s.AllowedSubscriptions) > 0 { + privateLinkToCreate.Properties.Visibility = &armnetwork.PrivateLinkServicePropertiesVisibility{ + Subscriptions: s.AllowedSubscriptions, + } + } + if len(s.AutoApprovedSubscriptions) > 0 { + privateLinkToCreate.Properties.AutoApproval = &armnetwork.PrivateLinkServicePropertiesAutoApproval{ + Subscriptions: s.AutoApprovedSubscriptions, + } + } + + return privateLinkToCreate, nil +} + +func isExistingUpToDate(existing armnetwork.PrivateLinkService, wanted armnetwork.PrivateLinkService) bool { + // NAT IP configuration is not checked as it cannot be changed. + + // Check load balancer configurations + wantedFrontendIDs := make([]*string, len(wanted.Properties.LoadBalancerFrontendIPConfigurations)) + for _, wantedFrontendIPConfig := range wanted.Properties.LoadBalancerFrontendIPConfigurations { + wantedFrontendIDs = append(wantedFrontendIDs, wantedFrontendIPConfig.ID) + } + existingFrontendIDs := make([]*string, len(existing.Properties.LoadBalancerFrontendIPConfigurations)) + for _, existingFrontendIPConfig := range existing.Properties.LoadBalancerFrontendIPConfigurations { + existingFrontendIDs = append(existingFrontendIDs, existingFrontendIPConfig.ID) + } + if !compareStringPointerSlicesUnordered(wantedFrontendIDs, existingFrontendIDs) { + return false + } + + // Check proxy protocol config + if !ptr.Equal(wanted.Properties.EnableProxyProtocol, existing.Properties.EnableProxyProtocol) { + return false + } + + // Check allowed subscriptions + if !compareStringPointerSlicesUnordered( + wanted.Properties.Visibility.Subscriptions, + existing.Properties.Visibility.Subscriptions) { + return false + } + + // Check auto-approved subscriptions + if !compareStringPointerSlicesUnordered( + wanted.Properties.AutoApproval.Subscriptions, + existing.Properties.AutoApproval.Subscriptions) { + return false + } + + return true +} + +func compareStringPointerSlicesUnordered(a, b []*string) bool { + if len(a) != len(b) { + return false + } + m := make(map[string]struct{}, len(a)) + for _, x := range a { + if x == nil { + continue + } + m[*x] = struct{}{} + } + for _, y := range b { + if y == nil { + continue + } + if _, ok := m[*y]; !ok { + return false + } + } + return true +} diff --git a/azure/services/privatelinks/spec_test.go b/azure/services/privatelinks/spec_test.go new file mode 100644 index 00000000000..8934ce49b31 --- /dev/null +++ b/azure/services/privatelinks/spec_test.go @@ -0,0 +1,634 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package privatelinks + +import ( + "context" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + . "github.com/onsi/gomega" + "k8s.io/utils/ptr" +) + +const ( + fakeRegion = "westeurope" + fakeSubscriptionID1 = "abcd" + fakeSubscriptionID2 = "efgh" + fakeClusterName = "my-cluster" + fakeVNetResourceGroup = fakeClusterName + fakeVNetName = fakeClusterName + "-vnet" + fakeSubnetName = fakeClusterName + "-node-subnet" + fakeLbName = fakeClusterName + "-internal-lb" + fakeLbIPConfigName1 = fakeLbName + "-frontend1" + fakeLbIPConfigName2 = fakeLbName + "-frontend2" + fakePrivateLinkName = "apiserver-privatelink" +) + +var ( + // fakePrivateLinkSpec1 is private link spec with: + // - 1 allowed subscription, + // - 1 auto-approved subscription, + // - disabled proxy protocol and + // - additional tag "hello:capz". + fakePrivateLinkSpec1 = PrivateLinkSpec{ + Name: fakePrivateLinkName, + ResourceGroup: fakeClusterName, + SubscriptionID: fakeSubscriptionID1, + Location: fakeRegion, + VNetResourceGroup: fakeVNetResourceGroup, + VNet: fakeVNetName, + NATIPConfiguration: []NATIPConfiguration{ + { + AllocationMethod: string(armnetwork.IPAllocationMethodDynamic), + Subnet: fakeSubnetName, + }, + }, + LoadBalancerName: fakeLbName, + LBFrontendIPConfigNames: []string{ + fakeLbIPConfigName1, + }, + AllowedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + }, + AutoApprovedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + }, + EnableProxyProtocol: ptr.To(false), + ClusterName: fakeClusterName, + AdditionalTags: map[string]string{ + "hello": "capz", + }, + } + + // fakePrivateLinkSpec2 is modified fakePrivateLinkSpec1 with following changes: + // - 1 added allowed subscription. + fakePrivateLinkSpec2 = PrivateLinkSpec{ + Name: fakePrivateLinkName, + ResourceGroup: fakeClusterName, + SubscriptionID: fakeSubscriptionID1, + Location: fakeRegion, + VNetResourceGroup: fakeVNetResourceGroup, + VNet: fakeVNetName, + NATIPConfiguration: []NATIPConfiguration{ + { + AllocationMethod: string(armnetwork.IPAllocationMethodDynamic), + Subnet: fakeSubnetName, + }, + }, + LoadBalancerName: fakeLbName, + LBFrontendIPConfigNames: []string{ + fakeLbIPConfigName1, + }, + AllowedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + AutoApprovedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + }, + EnableProxyProtocol: ptr.To(false), + ClusterName: fakeClusterName, + AdditionalTags: map[string]string{ + "hello": "capz", + }, + } + + // fakePrivateLinkSpec3 is modified fakePrivateLinkSpec2 with following changes: + // - 1 added auto-approved subscription. + fakePrivateLinkSpec3 = PrivateLinkSpec{ + Name: fakePrivateLinkName, + ResourceGroup: fakeClusterName, + SubscriptionID: fakeSubscriptionID1, + Location: fakeRegion, + VNetResourceGroup: fakeVNetResourceGroup, + VNet: fakeVNetName, + NATIPConfiguration: []NATIPConfiguration{ + { + AllocationMethod: string(armnetwork.IPAllocationMethodDynamic), + Subnet: fakeSubnetName, + }, + }, + LoadBalancerName: fakeLbName, + LBFrontendIPConfigNames: []string{ + fakeLbIPConfigName1, + }, + AllowedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + AutoApprovedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + EnableProxyProtocol: ptr.To(false), + ClusterName: fakeClusterName, + AdditionalTags: map[string]string{ + "hello": "capz", + }, + } + + // fakePrivateLinkSpec4 is modified fakePrivateLinkSpec3 with following changes: + // - enabled proxy protocol. + fakePrivateLinkSpec4 = PrivateLinkSpec{ + Name: fakePrivateLinkName, + ResourceGroup: fakeClusterName, + SubscriptionID: fakeSubscriptionID1, + Location: fakeRegion, + VNetResourceGroup: fakeVNetResourceGroup, + VNet: fakeVNetName, + NATIPConfiguration: []NATIPConfiguration{ + { + AllocationMethod: string(armnetwork.IPAllocationMethodDynamic), + Subnet: fakeSubnetName, + }, + }, + LoadBalancerName: fakeLbName, + LBFrontendIPConfigNames: []string{ + fakeLbIPConfigName1, + }, + AllowedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + AutoApprovedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + EnableProxyProtocol: ptr.To(true), + ClusterName: fakeClusterName, + AdditionalTags: map[string]string{ + "hello": "capz", + }, + } + + // fakePrivateLinkSpec5 is modified fakePrivateLinkSpec4 with following changes: + // - changed LB frontend config name. + fakePrivateLinkSpec5 = PrivateLinkSpec{ + Name: fakePrivateLinkName, + ResourceGroup: fakeClusterName, + SubscriptionID: fakeSubscriptionID1, + Location: fakeRegion, + VNetResourceGroup: fakeVNetResourceGroup, + VNet: fakeVNetName, + NATIPConfiguration: []NATIPConfiguration{ + { + AllocationMethod: string(armnetwork.IPAllocationMethodDynamic), + Subnet: fakeSubnetName, + }, + }, + LoadBalancerName: fakeLbName, + LBFrontendIPConfigNames: []string{ + fakeLbIPConfigName2, + }, + AllowedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + AutoApprovedSubscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + EnableProxyProtocol: ptr.To(true), + ClusterName: fakeClusterName, + AdditionalTags: map[string]string{ + "hello": "capz", + }, + } + + // fakePrivateLink1 is Azure PrivateLinkService that corresponds to fakePrivateLinkSpec1. + fakePrivateLink1 = armnetwork.PrivateLinkService{ + Name: ptr.To(fakePrivateLinkName), + Location: ptr.To(fakeRegion), + Properties: &armnetwork.PrivateLinkServiceProperties{ + IPConfigurations: []*armnetwork.PrivateLinkServiceIPConfiguration{ + { + Name: ptr.To(fmt.Sprintf("%s-natipconfig-1", fakeSubnetName)), + Properties: &armnetwork.PrivateLinkServiceIPConfigurationProperties{ + Subnet: &armnetwork.Subnet{ + ID: ptr.To( + fmt.Sprintf( + "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", + fakeSubscriptionID1, + fakeVNetResourceGroup, + fakeVNetName, + fakeSubnetName)), + }, + PrivateIPAllocationMethod: ptr.To(armnetwork.IPAllocationMethodDynamic), + Primary: ptr.To(true), + }, + }, + }, + LoadBalancerFrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ + { + ID: ptr.To( + fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", + fakeSubscriptionID1, + fakeClusterName, + fakeLbName, + fakeLbIPConfigName1)), + }, + }, + Visibility: &armnetwork.PrivateLinkServicePropertiesVisibility{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + }, + }, + AutoApproval: &armnetwork.PrivateLinkServicePropertiesAutoApproval{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + }, + }, + EnableProxyProtocol: ptr.To(false), + }, + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_" + fakeClusterName: ptr.To("owned"), + "Name": ptr.To(fakePrivateLinkName), + "hello": ptr.To("capz"), + }, + } + + // fakePrivateLink2 is Azure PrivateLinkService that corresponds to fakePrivateLinkSpec2. + fakePrivateLink2 = armnetwork.PrivateLinkService{ + Name: ptr.To(fakePrivateLinkName), + Location: ptr.To(fakeRegion), + Properties: &armnetwork.PrivateLinkServiceProperties{ + IPConfigurations: []*armnetwork.PrivateLinkServiceIPConfiguration{ + { + Name: ptr.To(fmt.Sprintf("%s-natipconfig-1", fakeSubnetName)), + Properties: &armnetwork.PrivateLinkServiceIPConfigurationProperties{ + Subnet: &armnetwork.Subnet{ + ID: ptr.To( + fmt.Sprintf( + "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", + fakeSubscriptionID1, + fakeVNetResourceGroup, + fakeVNetName, + fakeSubnetName)), + }, + PrivateIPAllocationMethod: ptr.To(armnetwork.IPAllocationMethodDynamic), + Primary: ptr.To(true), + }, + }, + }, + LoadBalancerFrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ + { + ID: ptr.To( + fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", + fakeSubscriptionID1, + fakeClusterName, + fakeLbName, + fakeLbIPConfigName1)), + }, + }, + Visibility: &armnetwork.PrivateLinkServicePropertiesVisibility{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + }, + AutoApproval: &armnetwork.PrivateLinkServicePropertiesAutoApproval{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + }, + }, + EnableProxyProtocol: ptr.To(false), + }, + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_" + fakeClusterName: ptr.To("owned"), + "Name": ptr.To(fakePrivateLinkName), + "hello": ptr.To("capz"), + }, + } + + // fakePrivateLink3 is Azure PrivateLinkService that corresponds to fakePrivateLinkSpec3. + fakePrivateLink3 = armnetwork.PrivateLinkService{ + Name: ptr.To(fakePrivateLinkName), + Location: ptr.To(fakeRegion), + Properties: &armnetwork.PrivateLinkServiceProperties{ + IPConfigurations: []*armnetwork.PrivateLinkServiceIPConfiguration{ + { + Name: ptr.To(fmt.Sprintf("%s-natipconfig-1", fakeSubnetName)), + Properties: &armnetwork.PrivateLinkServiceIPConfigurationProperties{ + Subnet: &armnetwork.Subnet{ + ID: ptr.To( + fmt.Sprintf( + "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", + fakeSubscriptionID1, + fakeVNetResourceGroup, + fakeVNetName, + fakeSubnetName)), + }, + PrivateIPAllocationMethod: ptr.To(armnetwork.IPAllocationMethodDynamic), + Primary: ptr.To(true), + }, + }, + }, + LoadBalancerFrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ + { + ID: ptr.To( + fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", + fakeSubscriptionID1, + fakeClusterName, + fakeLbName, + fakeLbIPConfigName1)), + }, + }, + Visibility: &armnetwork.PrivateLinkServicePropertiesVisibility{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + }, + AutoApproval: &armnetwork.PrivateLinkServicePropertiesAutoApproval{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + }, + EnableProxyProtocol: ptr.To(false), + }, + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_" + fakeClusterName: ptr.To("owned"), + "Name": ptr.To(fakePrivateLinkName), + "hello": ptr.To("capz"), + }, + } + + // fakePrivateLink4 is Azure PrivateLinkService that corresponds to fakePrivateLinkSpec4. + fakePrivateLink4 = armnetwork.PrivateLinkService{ + Name: ptr.To(fakePrivateLinkName), + Location: ptr.To(fakeRegion), + Properties: &armnetwork.PrivateLinkServiceProperties{ + IPConfigurations: []*armnetwork.PrivateLinkServiceIPConfiguration{ + { + Name: ptr.To(fmt.Sprintf("%s-natipconfig-1", fakeSubnetName)), + Properties: &armnetwork.PrivateLinkServiceIPConfigurationProperties{ + Subnet: &armnetwork.Subnet{ + ID: ptr.To( + fmt.Sprintf( + "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", + fakeSubscriptionID1, + fakeVNetResourceGroup, + fakeVNetName, + fakeSubnetName)), + }, + PrivateIPAllocationMethod: ptr.To(armnetwork.IPAllocationMethodDynamic), + Primary: ptr.To(true), + }, + }, + }, + LoadBalancerFrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ + { + ID: ptr.To( + fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", + fakeSubscriptionID1, + fakeClusterName, + fakeLbName, + fakeLbIPConfigName1)), + }, + }, + Visibility: &armnetwork.PrivateLinkServicePropertiesVisibility{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + }, + AutoApproval: &armnetwork.PrivateLinkServicePropertiesAutoApproval{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + }, + EnableProxyProtocol: ptr.To(true), + }, + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_" + fakeClusterName: ptr.To("owned"), + "Name": ptr.To(fakePrivateLinkName), + "hello": ptr.To("capz"), + }, + } + + // fakePrivateLink5 is Azure PrivateLinkService that corresponds to fakePrivateLinkSpec5. + fakePrivateLink5 = armnetwork.PrivateLinkService{ + Name: ptr.To(fakePrivateLinkName), + Location: ptr.To(fakeRegion), + Properties: &armnetwork.PrivateLinkServiceProperties{ + IPConfigurations: []*armnetwork.PrivateLinkServiceIPConfiguration{ + { + Name: ptr.To(fmt.Sprintf("%s-natipconfig-1", fakeSubnetName)), + Properties: &armnetwork.PrivateLinkServiceIPConfigurationProperties{ + Subnet: &armnetwork.Subnet{ + ID: ptr.To( + fmt.Sprintf( + "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", + fakeSubscriptionID1, + fakeVNetResourceGroup, + fakeVNetName, + fakeSubnetName)), + }, + PrivateIPAllocationMethod: ptr.To(armnetwork.IPAllocationMethodDynamic), + Primary: ptr.To(true), + }, + }, + }, + LoadBalancerFrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ + { + ID: ptr.To( + fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", + fakeSubscriptionID1, + fakeClusterName, + fakeLbName, + fakeLbIPConfigName2)), + }, + }, + Visibility: &armnetwork.PrivateLinkServicePropertiesVisibility{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + }, + AutoApproval: &armnetwork.PrivateLinkServicePropertiesAutoApproval{ + Subscriptions: []*string{ + ptr.To(fakeSubscriptionID1), + ptr.To(fakeSubscriptionID2), + }, + }, + EnableProxyProtocol: ptr.To(true), + }, + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_" + fakeClusterName: ptr.To("owned"), + "Name": ptr.To(fakePrivateLinkName), + "hello": ptr.To("capz"), + }, + } +) + +func TestEqualStringSlicesPtrIgnoreOrder(t *testing.T) { + testCases := []struct { + name string + value1 []*string + value2 []*string + expected bool + }{ + { + name: "Nil slices", + value1: nil, + value2: nil, + expected: true, + }, + { + name: "Empty slices", + value1: []*string{}, + value2: []*string{}, + expected: true, + }, + { + name: "Nil and empty slice", + value1: nil, + value2: []*string{}, + expected: true, + }, + { + name: "Same slices with 1 element", + value1: []*string{ptr.To("a")}, + value2: []*string{ptr.To("a")}, + expected: true, + }, + { + name: "Same slices with 3 elements in same order", + value1: []*string{ptr.To("a"), ptr.To("b"), ptr.To("c")}, + value2: []*string{ptr.To("a"), ptr.To("b"), ptr.To("c")}, + expected: true, + }, + { + name: "Same slices with 3 elements in different order", + value1: []*string{ptr.To("a"), ptr.To("b"), ptr.To("c")}, + value2: []*string{ptr.To("c"), ptr.To("a"), ptr.To("b")}, + expected: true, + }, + { + name: "Different slices with 1 element", + value1: []*string{ptr.To("a")}, + value2: []*string{ptr.To("b")}, + expected: false, + }, + { + name: "Different slices with 3 elements", + value1: []*string{ptr.To("a"), ptr.To("b"), ptr.To("c")}, + value2: []*string{ptr.To("a"), ptr.To("b"), ptr.To("d")}, + expected: false, + }, + { + name: "Slices with different lengths", + value1: []*string{ptr.To("a"), ptr.To("b"), ptr.To("c")}, + value2: []*string{ptr.To("a"), ptr.To("b"), ptr.To("c"), ptr.To("d")}, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result := compareStringPointerSlicesUnordered(tc.value1, tc.value2) + g.Expect(tc.expected).To(Equal(result)) + }) + } +} + +func TestParameters(t *testing.T) { + testcases := []struct { + name string + spec PrivateLinkSpec + existing any + expect func(g *WithT, result any) + expectedError string + }{ + { + name: "PrivateLink does not exist", + spec: fakePrivateLinkSpec1, + existing: nil, + expect: func(g *WithT, result any) { + g.Expect(result).To(BeAssignableToTypeOf(armnetwork.PrivateLinkService{})) + g.Expect(result).To(Equal(fakePrivateLink1)) + }, + }, + { + name: "PrivateLink already exists with the same config", + spec: fakePrivateLinkSpec1, + existing: fakePrivateLink1, + expect: func(g *WithT, result any) { + g.Expect(result).To(BeNil()) + }, + }, + { + name: "PrivateLink changed and added one new allowed subscription", + spec: fakePrivateLinkSpec2, // spec with 2 allowed subscriptions + existing: fakePrivateLink1, // existing private link with 1 allowed subscription + expect: func(g *WithT, result any) { + g.Expect(result).To(BeAssignableToTypeOf(armnetwork.PrivateLinkService{})) + g.Expect(result).To(Equal(fakePrivateLink2)) // expects (updated) private link with 2 allowed subscriptions + }, + }, + { + name: "PrivateLink changed and added one new auto-approved subscription", + spec: fakePrivateLinkSpec3, // spec with 2 auto-approved subscriptions + existing: fakePrivateLink2, // existing private link with 1 auto-approved subscription + expect: func(g *WithT, result any) { + g.Expect(result).To(BeAssignableToTypeOf(armnetwork.PrivateLinkService{})) + g.Expect(result).To(Equal(fakePrivateLink3)) // expects (updated) private link with 2 auto-approved subscriptions + }, + }, + { + name: "PrivateLink changed and enabled proxy protocol", + spec: fakePrivateLinkSpec4, // spec with enabled proxy protocol + existing: fakePrivateLink3, // existing private link with disabled proxy protocol + expect: func(g *WithT, result any) { + g.Expect(result).To(BeAssignableToTypeOf(armnetwork.PrivateLinkService{})) + g.Expect(result).To(Equal(fakePrivateLink4)) // expects (updated) private link with enabled proxy protocol + }, + }, + { + name: "PrivateLink changed LB frontend config name", + spec: fakePrivateLinkSpec5, // spec with changed LB frontend config name + existing: fakePrivateLink4, // existing private link with old LB frontend config name + expect: func(g *WithT, result any) { + g.Expect(result).To(BeAssignableToTypeOf(armnetwork.PrivateLinkService{})) + g.Expect(result).To(Equal(fakePrivateLink5)) // expects (updated) private link with changed LB frontend config name + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result, err := tc.spec.Parameters(context.TODO(), tc.existing) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + tc.expect(g, result) + }) + } +} diff --git a/azure/services/subnets/spec.go b/azure/services/subnets/spec.go index d2ee840b60f..b8de87fd6d0 100644 --- a/azure/services/subnets/spec.go +++ b/azure/services/subnets/spec.go @@ -31,17 +31,18 @@ import ( // SubnetSpec defines the specification for a Subnet. type SubnetSpec struct { - Name string - ResourceGroup string - SubscriptionID string - CIDRs []string - VNetName string - VNetResourceGroup string - IsVNetManaged bool - RouteTableName string - SecurityGroupName string - NatGatewayName string - ServiceEndpoints infrav1.ServiceEndpoints + Name string + ResourceGroup string + SubscriptionID string + CIDRs []string + VNetName string + VNetResourceGroup string + IsVNetManaged bool + RouteTableName string + SecurityGroupName string + NatGatewayName string + ServiceEndpoints infrav1.ServiceEndpoints + UsedForPrivateLinkNATIP bool } // ResourceRef implements azure.ASOResourceSpecGetter. @@ -74,6 +75,12 @@ func (s *SubnetSpec) Parameters(_ context.Context, existing *asonetworkv1.Virtua subnet.Spec.AddressPrefix = &s.CIDRs[0] } + if s.UsedForPrivateLinkNATIP { + // Disable PrivateLinkServiceNetworkPolicies only if the subnet is used for private link NAT IP in the + // AzureCluster spec, otherwise do not set any value here so the existing settings is not affected. + subnet.Spec.PrivateLinkServiceNetworkPolicies = ptr.To(asonetworkv1.SubnetPropertiesFormat_PrivateLinkServiceNetworkPolicies_Disabled) + } + if s.RouteTableName != "" { subnet.Spec.RouteTable = &asonetworkv1.RouteTableSpec_VirtualNetworks_Subnet_SubResourceEmbedded{ Reference: &genruntime.ResourceReference{ diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml index e42c11a7ad9..7a79b608b2d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml @@ -757,6 +757,67 @@ spec: type: integer name: type: string + privateLinks: + description: PrivateLinks to the load balancer (max 8 private + links). + items: + description: PrivateLink configures an Azure private link. + properties: + allowedSubscriptions: + description: AllowedSubscriptions is a list of subscriptions + from which the private link can be accessed. + items: + type: string + type: array + autoApprovedSubscriptions: + description: |- + AutoApprovedSubscriptions is a list of subscription for which the connections to private link are automatically + approved. + items: + type: string + type: array + enableProxyProtocol: + description: EnableProxyProtocol indicates whether the + private link service is enabled for proxy protocol + or not. + type: boolean + lbFrontendIPConfigNames: + description: |- + LBFrontendIPConfigNames are the names of the load balancer FrontendIP to which the private link will forward + requests. The specified frontend IP configs must have the private IP set. + items: + type: string + type: array + name: + description: Name of the private link. + type: string + natIPConfigurations: + description: NATIPConfigurations specify up to 8 NAT + IP configurations for the private link. + items: + description: PrivateLinkNATIPConfiguration specifies + NAT IP configuration for the private link. + properties: + allocationMethod: + description: 'AllocationMethod specifies how the + private link NAT IPs are allocated: "Static" + or "Dynamic".' + type: string + privateIPAddress: + type: string + subnet: + description: Subnet from which the IP is allocated. + type: string + required: + - allocationMethod + - subnet + type: object + type: array + required: + - lbFrontendIPConfigNames + - natIPConfigurations + type: object + type: array sku: description: SKU defines an Azure load balancer SKU. type: string @@ -840,6 +901,67 @@ spec: type: integer name: type: string + privateLinks: + description: PrivateLinks to the load balancer (max 8 private + links). + items: + description: PrivateLink configures an Azure private link. + properties: + allowedSubscriptions: + description: AllowedSubscriptions is a list of subscriptions + from which the private link can be accessed. + items: + type: string + type: array + autoApprovedSubscriptions: + description: |- + AutoApprovedSubscriptions is a list of subscription for which the connections to private link are automatically + approved. + items: + type: string + type: array + enableProxyProtocol: + description: EnableProxyProtocol indicates whether the + private link service is enabled for proxy protocol + or not. + type: boolean + lbFrontendIPConfigNames: + description: |- + LBFrontendIPConfigNames are the names of the load balancer FrontendIP to which the private link will forward + requests. The specified frontend IP configs must have the private IP set. + items: + type: string + type: array + name: + description: Name of the private link. + type: string + natIPConfigurations: + description: NATIPConfigurations specify up to 8 NAT + IP configurations for the private link. + items: + description: PrivateLinkNATIPConfiguration specifies + NAT IP configuration for the private link. + properties: + allocationMethod: + description: 'AllocationMethod specifies how the + private link NAT IPs are allocated: "Static" + or "Dynamic".' + type: string + privateIPAddress: + type: string + subnet: + description: Subnet from which the IP is allocated. + type: string + required: + - allocationMethod + - subnet + type: object + type: array + required: + - lbFrontendIPConfigNames + - natIPConfigurations + type: object + type: array sku: description: SKU defines an Azure load balancer SKU. type: string @@ -922,6 +1044,67 @@ spec: type: integer name: type: string + privateLinks: + description: PrivateLinks to the load balancer (max 8 private + links). + items: + description: PrivateLink configures an Azure private link. + properties: + allowedSubscriptions: + description: AllowedSubscriptions is a list of subscriptions + from which the private link can be accessed. + items: + type: string + type: array + autoApprovedSubscriptions: + description: |- + AutoApprovedSubscriptions is a list of subscription for which the connections to private link are automatically + approved. + items: + type: string + type: array + enableProxyProtocol: + description: EnableProxyProtocol indicates whether the + private link service is enabled for proxy protocol + or not. + type: boolean + lbFrontendIPConfigNames: + description: |- + LBFrontendIPConfigNames are the names of the load balancer FrontendIP to which the private link will forward + requests. The specified frontend IP configs must have the private IP set. + items: + type: string + type: array + name: + description: Name of the private link. + type: string + natIPConfigurations: + description: NATIPConfigurations specify up to 8 NAT + IP configurations for the private link. + items: + description: PrivateLinkNATIPConfiguration specifies + NAT IP configuration for the private link. + properties: + allocationMethod: + description: 'AllocationMethod specifies how the + private link NAT IPs are allocated: "Static" + or "Dynamic".' + type: string + privateIPAddress: + type: string + subnet: + description: Subnet from which the IP is allocated. + type: string + required: + - allocationMethod + - subnet + type: object + type: array + required: + - lbFrontendIPConfigNames + - natIPConfigurations + type: object + type: array sku: description: SKU defines an Azure load balancer SKU. type: string diff --git a/controllers/azurecluster_reconciler.go b/controllers/azurecluster_reconciler.go index 93775d166f9..0bb5d6ddebc 100644 --- a/controllers/azurecluster_reconciler.go +++ b/controllers/azurecluster_reconciler.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure/services/natgateways" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privatedns" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/privatelinks" "sigs.k8s.io/cluster-api-provider-azure/azure/services/publicips" "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus" "sigs.k8s.io/cluster-api-provider-azure/azure/services/routetables" @@ -75,6 +76,10 @@ func newAzureClusterService(scope *scope.ClusterScope) (*azureClusterService, er if err != nil { return nil, err } + privateLinksSvc, err := privatelinks.New(scope) + if err != nil { + return nil, err + } vnetPeeringsSvc, err := vnetpeerings.New(scope) if err != nil { return nil, err @@ -96,6 +101,7 @@ func newAzureClusterService(scope *scope.ClusterScope) (*azureClusterService, er vnetPeeringsSvc, loadbalancersSvc, privateDNSSvc, + privateLinksSvc, privateendpoints.New(scope), bastionhosts.New(scope), }, diff --git a/docs/book/src/self-managed/api-server-endpoint.md b/docs/book/src/self-managed/api-server-endpoint.md index 0016505c0db..7a124d9ae2f 100644 --- a/docs/book/src/self-managed/api-server-endpoint.md +++ b/docs/book/src/self-managed/api-server-endpoint.md @@ -8,7 +8,7 @@ CAPZ supports two load balancer types, `Public` and `Internal`. `Public`, which is also the default, means that your API Server Load Balancer will have a publicly accessible IP address. This Load Balancer type supports a "public cluster" configuration, which load balances internet source traffic to the apiserver across the cluster's control plane nodes. -`Internal` means that the API Server endpoint will only be accessible from within the cluster's virtual network (or peered VNets). This configuration supports a "private cluster" configuration, which load balances internal VNET source traffic to the apiserver across the cluster's control plane nodes. +`Internal` means that the API Server endpoint will only be accessible from within the cluster's virtual network (or peered VNets). This configuration supports a "private cluster" configuration, which load balances internal VNET source traffic to the apiserver across the cluster's control plane nodes. Here it is also possible to expose API Server load balancer via a private link, as the source of traffic that is coming via private link will be NAT IP addresses chosen from a workload cluster subnet. For a more complete "private cluster" template example, you may refer to [this reference template](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/cluster-template-private.yaml) that the capz project maintains. @@ -73,6 +73,45 @@ spec: privateIP: 172.16.0.100 ``` +### Private link + +Here is an example of configuring the API Server LB with a private link + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: my-private-cluster + namespace: default +spec: + location: eastus + networkSpec: + vnet: + name: my-vnet + cidrBlocks: + - 172.16.0.0/16 + subnets: + - name: my-subnet-cp + role: control-plane + cidrBlocks: + - 172.16.0.0/24 + - name: my-subnet-node + role: node + cidrBlocks: + - 172.16.2.0/24 + apiServerLB: + type: Internal + frontendIPs: + - name: lb-private-ip-frontend + privateIP: 172.16.0.100 + privateLinks: + - lbFrontendIPConfigNames: + - lb-private-ip-frontend + natIPConfigurations: + - allocationMethod: Dynamic + subnet: my-subnet-node +``` + ### Public IP When using an api server load balancer of type `Public`, a dynamic public IP address will be created, along with a unique FQDN. diff --git a/internal/webhooks/azurecluster_validation.go b/internal/webhooks/azurecluster_validation.go index 1dbca65b9d5..fc94d458c15 100644 --- a/internal/webhooks/azurecluster_validation.go +++ b/internal/webhooks/azurecluster_validation.go @@ -48,6 +48,8 @@ const ( loadBalancerRegex = `^[-\w\._]+$` // MaxLoadBalancerOutboundIPs is the maximum number of outbound IPs in a Standard LoadBalancer frontend configuration. MaxLoadBalancerOutboundIPs = 16 + // MaxLoadBalancerPrivateLinks is the maximum number of private links attached to a Standard Loadbalancer. + MaxLoadBalancerPrivateLinks = 8 // MinLBIdleTimeoutInMinutes is the minimum number of minutes for the LB idle timeout. MinLBIdleTimeoutInMinutes = 4 // MaxLBIdleTimeoutInMinutes is the maximum number of minutes for the LB idle timeout. @@ -62,6 +64,7 @@ const ( serviceEndpointLocationRegexPattern = `^([a-z]{1,42}\d{0,5}|[*])$` // described in https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules. privateEndpointRegex = `^[-\w\._]+$` + privateLinkRegex = `^[-\w\._]+$` // resource ID Pattern. resourceIDPattern = `(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/(.+?)/(.+?)/(.+)` ) @@ -183,7 +186,7 @@ func validateNetworkSpec(controlPlaneEnabled bool, networkSpec infrav1.NetworkSp } cidrBlocks = controlPlaneSubnet.CIDRBlocks - allErrs = append(allErrs, validateAPIServerLB(networkSpec.APIServerLB, old.APIServerLB, cidrBlocks, fldPath.Child("apiServerLB"))...) + allErrs = append(allErrs, validateAPIServerLB(networkSpec.APIServerLB, old.APIServerLB, networkSpec.Subnets, cidrBlocks, fldPath.Child("apiServerLB"))...) } var needOutboundLB bool @@ -392,7 +395,7 @@ func validateSecurityRule(rule infrav1.SecurityRule, fldPath *field.Path) (allEr return allErrs } -func validateAPIServerLB(lb *infrav1.LoadBalancerSpec, old *infrav1.LoadBalancerSpec, cidrs []string, fldPath *field.Path) field.ErrorList { +func validateAPIServerLB(lb *infrav1.LoadBalancerSpec, old *infrav1.LoadBalancerSpec, subnets infrav1.Subnets, cidrs []string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList lbClassSpec := lb.LoadBalancerClassSpec @@ -463,6 +466,10 @@ func validateAPIServerLB(lb *infrav1.LoadBalancerSpec, old *infrav1.LoadBalancer } } } + + // Validate private links to load balancer + allErrs = append(allErrs, validateLBPrivateLinks(lb, old, subnets, fldPath)...) + return allErrs } @@ -749,7 +756,7 @@ func validatePrivateEndpoints(privateEndpointSpecs []infrav1.PrivateEndpointSpec } for _, privateIP := range pe.PrivateIPAddresses { - if err := validatePrivateEndpointIPAddress(privateIP, subnetCIDRs, fldPath.Index(i).Child("privateIPAddresses")); err != nil { + if err := validateIPAddress(privateIP, subnetCIDRs, fldPath.Index(i).Child("privateIPAddresses"), pe.Name); err != nil { allErrs = append(allErrs, err) } } @@ -758,6 +765,152 @@ func validatePrivateEndpoints(privateEndpointSpecs []infrav1.PrivateEndpointSpec return allErrs } +func validateLBPrivateLinks(lb *infrav1.LoadBalancerSpec, oldLb *infrav1.LoadBalancerSpec, subnets infrav1.Subnets, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + if lb.Type != infrav1.Internal && len(lb.PrivateLinks) > 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("privateLinks"), lb.PrivateLinks, "private links can be added only to an internal load balancer")) + return allErrs + } + + if len(lb.PrivateLinks) > MaxLoadBalancerPrivateLinks { + allErrs = append(allErrs, field.Invalid(fldPath.Child("privateLinks"), lb.PrivateLinks, "maximum number of private links per load balancer is 8 (Azure limit)")) + return allErrs + } + + oldPrivateLinksMap := make(map[string]infrav1.PrivateLink) + if oldLb != nil { + for _, oldPrivateLink := range oldLb.PrivateLinks { + oldPrivateLinksMap[oldPrivateLink.Name] = oldPrivateLink + } + } + + for i, pl := range lb.PrivateLinks { + if err := validatePrivateLinkName(pl.Name, fldPath.Child("privateLinks").Index(i).Child("name")); err != nil { + allErrs = append(allErrs, err) + } + + // validate LB front end names that the private link is using: + // - LB front end names cannot be empty + // - LB front end names must be those that are specified in LoadBalancerSpec.FrontendIPs + // - private link cannot have duplicate entries for LB front end names + if len(pl.LBFrontendIPConfigNames) == 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("privateLinks").Index(i).Child("lbFrontendIPConfigNames"), pl.LBFrontendIPConfigNames, "LBFrontendIPConfigNames cannot be empty")) + } else { + lbFrontendIPNamesErrorAdded := map[string]bool{} + lbFrontendIPNamesCount := map[string]int{} + for j, lbFrontendIPName := range pl.LBFrontendIPConfigNames { + lbFrontendIPNameValid := false + for _, apiLbFrontendIP := range lb.FrontendIPs { + if lbFrontendIPName == apiLbFrontendIP.Name { + lbFrontendIPNameValid = true + break + } + } + if !lbFrontendIPNameValid { + if _, errorAlreadyAdded := lbFrontendIPNamesErrorAdded[lbFrontendIPName]; !errorAlreadyAdded { + allErrs = append( + allErrs, + field.Invalid( + fldPath.Child("privateLinks").Index(i).Child("lbFrontendIPConfigNames").Index(j), + lbFrontendIPName, + "LBFrontendIPConfigName must exist in the API server LoadBalancerSpec.FrontendIPs")) + lbFrontendIPNamesErrorAdded[lbFrontendIPName] = true + } + break + } + lbFrontendIPNamesCount[lbFrontendIPName]++ + } + + for lbFrontendIPConfigName, count := range lbFrontendIPNamesCount { + if count > 1 { + allErrs = append( + allErrs, + field.Invalid( + fldPath.Child("privateLinks").Index(i).Child("lbFrontendIPConfigNames"), + pl.LBFrontendIPConfigNames, + fmt.Sprintf("LBFrontendIPConfigNames cannot have duplicate entries (%s has been specified %d times)", lbFrontendIPConfigName, count))) + } + } + } + + // validate NAT IP configurations for the private link + // - NAT IP configurations cannot be empty + // - there can be maximum 8 NAT IP configurations (Azure limit) + // - NAT IP configuration must use valid subnet from AzureCluster.Spec.NetworkSpec + switch { + case len(pl.NATIPConfigurations) == 0: + allErrs = append( + allErrs, + field.Invalid( + fldPath.Child("privateLinks").Index(i).Child("natIPConfigurations"), + pl.NATIPConfigurations, + "NATIPConfigurations cannot be empty")) + case len(pl.NATIPConfigurations) > 8: + allErrs = append( + allErrs, + field.Invalid( + fldPath.Child("privateLinks").Index(i).Child("natIPConfigurations"), + pl.NATIPConfigurations, + "maximum number of NAT IP Configurations is 8 (Azure limit)")) + default: + var subnetCIDRs []string + for _, subnet := range subnets { + subnetCIDRs = append(subnetCIDRs, subnet.CIDRBlocks...) + } + // validate that NAT IP configurations are correct + for j, natIPConfig := range pl.NATIPConfigurations { + if natIPConfig.Subnet == "" { + allErrs = append( + allErrs, + field.Invalid( + fldPath.Child("privateLinks").Index(i).Child("natIPConfigurations").Index(j), + pl.NATIPConfigurations[j], + "NATIPConfiguration must specify an existing subnet name")) + break + } + + usesValidSubnet := false + for _, subnet := range subnets { + if natIPConfig.Subnet == subnet.Name { + usesValidSubnet = true + } + } + if !usesValidSubnet { + allErrs = append( + allErrs, + field.Invalid( + fldPath.Child("privateLinks").Index(i).Child("natIPConfigurations").Index(j).Child("subnet"), + pl.NATIPConfigurations[j].Subnet, + fmt.Sprintf("NATIPConfiguration must use existing subnet (subnet %s not specified in AzureCluster resource)", natIPConfig.Subnet))) + } + if natIPConfig.AllocationMethod == "Static" { + err := validateIPAddress( + natIPConfig.PrivateIPAddress, + subnetCIDRs, + fldPath.Child("privateLinks").Index(i).Child("natIPConfigurations").Index(j).Child("privateIPAddress"), + pl.Name) + if err != nil { + allErrs = append(allErrs, err) + } + } + } + + // validate that NAT IP configurations have not changed + if oldPrivateLink, ok := oldPrivateLinksMap[pl.Name]; ok && !reflect.DeepEqual(pl.NATIPConfigurations, oldPrivateLink.NATIPConfigurations) { + allErrs = append( + allErrs, + field.Invalid( + fldPath.Child("privateLinks").Index(i).Child("natIPConfigurations"), + pl.NATIPConfigurations, + "NATIPConfigurations cannot be modified")) + } + } + } + + return allErrs +} + // validatePrivateEndpointName validates the Name of a Private Endpoint. func validatePrivateEndpointName(name string, fldPath *field.Path) *field.Error { if name == "" { @@ -786,12 +939,12 @@ func validatePrivateEndpointPrivateLinkServiceConnection(privateLinkServiceConne return nil } -// validatePrivateEndpointIPAddress validates a Private Endpoint IP Address. -func validatePrivateEndpointIPAddress(address string, cidrs []string, fldPath *field.Path) *field.Error { +// validateIPAddress validates a Private Endpoint or Private Link IP Address. +func validateIPAddress(address string, cidrs []string, fldPath *field.Path, ownerName string) *field.Error { ip := net.ParseIP(address) if ip == nil { return field.Invalid(fldPath, address, - "Private Endpoint IP address isn't a valid IPv4 or IPv6 address") + fmt.Sprintf("%s IP address isn't a valid IPv4 or IPv6 address", ownerName)) } for _, cidr := range cidrs { @@ -802,7 +955,20 @@ func validatePrivateEndpointIPAddress(address string, cidrs []string, fldPath *f } return field.Invalid(fldPath, address, - fmt.Sprintf("Private Endpoint IP address needs to be in subnet range (%s)", cidrs)) + fmt.Sprintf("%s IP address needs to be in subnet range (%s)", ownerName, cidrs)) +} + +// validatePrivateLinkName validates the Name of a Private Link. +func validatePrivateLinkName(name string, fldPath *field.Path) *field.Error { + if name == "" { + return field.Invalid(fldPath, name, "name of private link cannot be empty") + } + + if success, _ := regexp.MatchString(privateLinkRegex, name); !success { + return field.Invalid(fldPath, name, + fmt.Sprintf("name of private link doesn't match regex %s", privateLinkRegex)) + } + return nil } // validateAzureClusterSubnetUpdate validates a ClusterSpec.NetworkSpec.Subnets for immutability. diff --git a/internal/webhooks/azurecluster_validation_test.go b/internal/webhooks/azurecluster_validation_test.go index 6ca46e99d17..efba583334d 100644 --- a/internal/webhooks/azurecluster_validation_test.go +++ b/internal/webhooks/azurecluster_validation_test.go @@ -1294,7 +1294,7 @@ func TestValidateAPIServerLB(t *testing.T) { if test.featureGate == feature.APIServerILB { featuregatetesting.SetFeatureGateDuringTest(t, feature.Gates, test.featureGate, true) } - err := validateAPIServerLB(test.lb, test.old, test.cpCIDRS, field.NewPath("apiServerLB")) + err := validateAPIServerLB(test.lb, test.old, infrav1.Subnets{}, test.cpCIDRS, field.NewPath("apiServerLB")) if test.wantErr { g.Expect(err).To(ContainElement(MatchError(test.expectedErr.Error()))) } else { @@ -1303,6 +1303,755 @@ func TestValidateAPIServerLB(t *testing.T) { }) } } + +func TestValidatePrivateLinks(t *testing.T) { + g := NewWithT(t) + + testcases := []struct { + name string + lb infrav1.LoadBalancerSpec + old infrav1.LoadBalancerSpec + subnets infrav1.Subnets + wantErr bool + expectedErr field.Error + }{ + { + name: "internal LB with a private link", + lb: infrav1.LoadBalancerSpec{ + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + FrontendIPClass: infrav1.FrontendIPClass{ + PrivateIPAddress: "10.1.0.3", + }, + }, + }, + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: "node-subnet", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + }, + }, + }, + wantErr: false, + }, + { + name: "internal LB with a private link with static NAT IP", + lb: infrav1.LoadBalancerSpec{ + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + FrontendIPClass: infrav1.FrontendIPClass{ + PrivateIPAddress: "10.1.0.3", + }, + }, + }, + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.10", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + CIDRBlocks: []string{ + "10.10.0.0/16", + }, + }, + }, + }, + wantErr: false, + }, + { + name: "public LB with a private link", + lb: infrav1.LoadBalancerSpec{ + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Public, + }, + Name: "my-public-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-public-lb-privatelink", + }, + }, + }, + subnets: infrav1.Subnets{}, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks", + BadValue: []infrav1.PrivateLink{ + { + Name: "my-public-lb-privatelink", + }, + }, + Detail: "private links can be added only to an internal load balancer", + }, + }, + { + name: "internal LB with more than 8 private links", + lb: infrav1.LoadBalancerSpec{ + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink-1", + }, + { + Name: "my-private-lb-privatelink-2", + }, + { + Name: "my-private-lb-privatelink-3", + }, + { + Name: "my-private-lb-privatelink-4", + }, + { + Name: "my-private-lb-privatelink-5", + }, + { + Name: "my-private-lb-privatelink-6", + }, + { + Name: "my-private-lb-privatelink-7", + }, + { + Name: "my-private-lb-privatelink-8", + }, + { + Name: "my-private-lb-privatelink-9", + }, + }, + }, + subnets: infrav1.Subnets{}, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks", + BadValue: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink-1", + }, + { + Name: "my-private-lb-privatelink-2", + }, + { + Name: "my-private-lb-privatelink-3", + }, + { + Name: "my-private-lb-privatelink-4", + }, + { + Name: "my-private-lb-privatelink-5", + }, + { + Name: "my-private-lb-privatelink-6", + }, + { + Name: "my-private-lb-privatelink-7", + }, + { + Name: "my-private-lb-privatelink-8", + }, + { + Name: "my-private-lb-privatelink-9", + }, + }, + Detail: "maximum number of private links per load balancer is 8 (Azure limit)", + }, + }, + { + name: "empty private link name", + lb: infrav1.LoadBalancerSpec{ + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "", + }, + }, + }, + subnets: infrav1.Subnets{}, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].name", + BadValue: "", + Detail: "name of private link cannot be empty", + }, + }, + { + name: "invalid private link name", + lb: infrav1.LoadBalancerSpec{ + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "apiserver-$", + }, + }, + }, + subnets: infrav1.Subnets{}, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].name", + BadValue: "apiserver-$", + Detail: "name of private link doesn't match regex ^[-\\w\\._]+$", + }, + }, + { + name: "private link does not have frontend configuration", + lb: infrav1.LoadBalancerSpec{ + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{}, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: "node-subnet", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].lbFrontendIPConfigNames", + BadValue: []string{}, + Detail: "LBFrontendIPConfigNames cannot be empty", + }, + }, + { + name: "private link with incorrect frontend configuration name", + lb: infrav1.LoadBalancerSpec{ + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + }, + }, + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "some-other-frontend-config", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: "node-subnet", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].lbFrontendIPConfigNames[0]", + BadValue: "some-other-frontend-config", + Detail: "LBFrontendIPConfigName must exist in the API server LoadBalancerSpec.FrontendIPs", + }, + }, + { + name: "private link with frontend configuration specified multiple times", + lb: infrav1.LoadBalancerSpec{ + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + }, + }, + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: "node-subnet", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].lbFrontendIPConfigNames", + BadValue: []string{ + "ip-1", + "ip-1", + }, + Detail: "LBFrontendIPConfigNames cannot have duplicate entries (ip-1 has been specified 2 times)", + }, + }, + { + name: "private link without NAT IP configurations", + lb: infrav1.LoadBalancerSpec{ + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + }, + }, + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{}, + }, + }, + }, + subnets: infrav1.Subnets{}, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].natIPConfigurations", + BadValue: []infrav1.PrivateLinkNATIPConfiguration{}, + Detail: "NATIPConfigurations cannot be empty", + }, + }, + { + name: "private link with more than 8 NAT IP configurations", + lb: infrav1.LoadBalancerSpec{ + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + }, + }, + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.11", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.12", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.13", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.14", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.15", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.16", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.17", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.18", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.19", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + CIDRBlocks: []string{ + "10.10.0.0/16", + }, + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].natIPConfigurations", + BadValue: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.11", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.12", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.13", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.14", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.15", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.16", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.17", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.18", + }, + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.10.0.19", + }, + }, + Detail: "maximum number of NAT IP Configurations is 8 (Azure limit)", + }, + }, + { + name: "private link with NAT IP config without specified subnet", + lb: infrav1.LoadBalancerSpec{ + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + }, + }, + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: "", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].natIPConfigurations[0]", + BadValue: infrav1.PrivateLinkNATIPConfiguration{ + AllocationMethod: "Dynamic", + Subnet: "", + }, + Detail: "NATIPConfiguration must specify an existing subnet name", + }, + }, + { + name: "private link with NAT IP config with non-existent subnet", + lb: infrav1.LoadBalancerSpec{ + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + }, + }, + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: "other-subnet", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].natIPConfigurations[0].subnet", + BadValue: "other-subnet", + Detail: "NATIPConfiguration must use existing subnet (subnet other-subnet not specified in AzureCluster resource)", + }, + }, + { + name: "private link with an out-of-range static NAT IP", + lb: infrav1.LoadBalancerSpec{ + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + }, + }, + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Static", + Subnet: "node-subnet", + PrivateIPAddress: "10.11.0.10", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + CIDRBlocks: []string{ + "10.10.0.0/16", + }, + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].natIPConfigurations[0].privateIPAddress", + BadValue: "10.11.0.10", + Detail: "my-private-lb-privatelink IP address needs to be in subnet range ([10.10.0.0/16])", + }, + }, + { + name: "private link NAT IP configuration cannot be modified", + old: infrav1.LoadBalancerSpec{ + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + }, + }, + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: "cp-subnet", + }, + }, + }, + }, + }, + lb: infrav1.LoadBalancerSpec{ + FrontendIPs: []infrav1.FrontendIP{ + { + Name: "ip-1", + }, + }, + LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{ + Type: infrav1.Internal, + }, + Name: "my-private-lb", + PrivateLinks: []infrav1.PrivateLink{ + { + Name: "my-private-lb-privatelink", + LBFrontendIPConfigNames: []string{ + "ip-1", + }, + NATIPConfigurations: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: "node-subnet", + }, + }, + }, + }, + }, + subnets: infrav1.Subnets{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "node-subnet", + }, + }, + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + Name: "cp-subnet", + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.privateLinks[0].natIPConfigurations", + BadValue: []infrav1.PrivateLinkNATIPConfiguration{ + { + AllocationMethod: "Dynamic", + Subnet: "node-subnet", + }, + }, + Detail: "NATIPConfigurations cannot be modified", + }, + }, + } + + for _, test := range testcases { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + err := validateLBPrivateLinks(&test.lb, &test.old, test.subnets, field.NewPath("apiServerLB")) + if test.wantErr { + g.Expect(err).To(ContainElement(MatchError(test.expectedErr.Error()))) + } else { + g.Expect(err).To(BeEmpty()) + } + }) + } +} + func TestPrivateDNSZoneName(t *testing.T) { testcases := []struct { name string diff --git a/templates/cluster-template-private-link.yaml b/templates/cluster-template-private-link.yaml new file mode 100644 index 00000000000..e1313e787fe --- /dev/null +++ b/templates/cluster-template-private-link.yaml @@ -0,0 +1,232 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + bastionSpec: + azureBastion: {} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + apiServerLB: + name: ${CLUSTER_NAME}-internal-lb + type: Internal + frontendIPs: + - name: ${CLUSTER_NAME}-internal-lb-frontend + privateIP: 10.0.0.100 + privateLinks: + - name: ${CLUSTER_NAME}-internal-lb-privatelink + lbFrontendIPConfigNames: + - ${CLUSTER_NAME}-internal-lb-frontend + natIPConfigurations: + - allocationMethod: Dynamic + subnet: control-plane-subnet + controlPlaneOutboundLB: + frontendIPsCount: 1 + nodeOutboundLB: + frontendIPsCount: 1 + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: + - if [ -f /tmp/kubeadm-join-config.yaml ] || [ -f /run/kubeadm/kubeadm-join-config.yaml + ]; then echo '127.0.0.1 apiserver.${CLUSTER_NAME}.capz.io apiserver' >> /etc/hosts; + fi + preKubeadmCommands: + - if [ -f /tmp/kubeadm.yaml ] || [ -f /run/kubeadm/kubeadm.yaml ]; then echo '127.0.0.1 apiserver.${CLUSTER_NAME}.capz.io + apiserver' >> /etc/hosts; fi + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal diff --git a/util/futures/getter.go b/util/futures/getter.go index 84e473d6c66..06bccbcdbd4 100644 --- a/util/futures/getter.go +++ b/util/futures/getter.go @@ -31,6 +31,23 @@ type Getter interface { GetFutures() infrav1.Futures } +// GetByServiceAndType returns the future for the specified service and future type, if the future does not exists, +// it returns nil. +func GetByServiceAndType(from Getter, service, futureType string) infrav1.Futures { + futures := from.GetFutures() + if futures == nil { + return nil + } + + var result infrav1.Futures + for _, f := range futures { + if f.ServiceName == service && f.Type == futureType { + result = append(result, f) + } + } + return result +} + // Get returns the future with the given name, if the future does not exists, // it returns nil. func Get(from Getter, name, service, futureType string) *infrav1.Future {