diff --git a/config/crd/bases/anywhere.eks.amazonaws.com_vspheredatacenterconfigs.yaml b/config/crd/bases/anywhere.eks.amazonaws.com_vspheredatacenterconfigs.yaml index 72f9e5fdeefe..0da62fc42448 100644 --- a/config/crd/bases/anywhere.eks.amazonaws.com_vspheredatacenterconfigs.yaml +++ b/config/crd/bases/anywhere.eks.amazonaws.com_vspheredatacenterconfigs.yaml @@ -83,6 +83,46 @@ spec: type: array insecure: type: boolean + ipPool: + description: |- + IPPool defines the IP pool configuration for static IP assignment to cluster nodes. + When specified, nodes will be assigned static IPs from this pool instead of using DHCP. + The CLI creates an InClusterIPPool resource from this configuration. + properties: + addresses: + description: |- + Addresses defines the IP addresses to include in the pool. + Supports ranges (e.g., "192.168.1.100-192.168.1.120"), + CIDR blocks (e.g., "192.168.1.0/24"), or individual IPs. + items: + type: string + minItems: 1 + type: array + gateway: + description: Gateway is the default gateway IP address for the + subnet. + type: string + name: + description: Name is the name for the generated InClusterIPPool + resource. + type: string + nameservers: + description: Nameservers is a list of DNS server IP addresses. + items: + type: string + type: array + prefix: + description: Prefix is the subnet prefix length (e.g., 24 for + /24 subnet). + maximum: 32 + minimum: 1 + type: integer + required: + - addresses + - gateway + - name + - prefix + type: object network: type: string server: diff --git a/internal/pkg/api/vsphere.go b/internal/pkg/api/vsphere.go index 1e599f72f44c..9e01551652fd 100644 --- a/internal/pkg/api/vsphere.go +++ b/internal/pkg/api/vsphere.go @@ -257,3 +257,24 @@ func RemoveEtcdVsphereMachineConfig() VSphereFiller { } } } + +// WithVSphereIPPool configures the IP pool for static IP assignment on the VSphereDatacenterConfig. +// This enables CAPI IPAM-based static IP allocation for vSphere clusters. +func WithVSphereIPPool(name string, addresses []string, prefix int, gateway string, nameservers []string) VSphereFiller { + return func(config VSphereConfig) { + config.datacenterConfig.Spec.IPPool = &anywherev1.IPPoolConfiguration{ + Name: name, + Addresses: addresses, + Prefix: prefix, + Gateway: gateway, + Nameservers: nameservers, + } + } +} + +// RemoveVSphereIPPool removes the IP pool configuration from the VSphereDatacenterConfig. +func RemoveVSphereIPPool() VSphereFiller { + return func(config VSphereConfig) { + config.datacenterConfig.Spec.IPPool = nil + } +} diff --git a/pkg/api/v1alpha1/cluster.go b/pkg/api/v1alpha1/cluster.go index c2781e8a115d..efad9f469850 100644 --- a/pkg/api/v1alpha1/cluster.go +++ b/pkg/api/v1alpha1/cluster.go @@ -1089,3 +1089,135 @@ func validateEksaVersion(clusterConfig *Cluster) error { return nil } + +// validateIPPoolConfig validates the IP pool configuration fields. +func validateIPPoolConfig(ipPool *IPPoolConfiguration) error { + if ipPool.Name == "" { + return errors.New("ipPool.name cannot be empty") + } + + if len(ipPool.Addresses) == 0 { + return errors.New("ipPool.addresses cannot be empty") + } + + for _, addr := range ipPool.Addresses { + if err := validateIPPoolAddress(addr); err != nil { + return fmt.Errorf("invalid ipPool.addresses entry %q: %v", addr, err) + } + } + + if ipPool.Prefix < 1 || ipPool.Prefix > 32 { + return fmt.Errorf("ipPool.prefix must be between 1 and 32, got %d", ipPool.Prefix) + } + + if err := validateIPPoolGatewayAndNameservers(ipPool); err != nil { + return err + } + + return nil +} + +// validateIPPoolGatewayAndNameservers validates gateway and nameserver IPs. +func validateIPPoolGatewayAndNameservers(ipPool *IPPoolConfiguration) error { + if ipPool.Gateway == "" { + return errors.New("ipPool.gateway cannot be empty") + } + if net.ParseIP(ipPool.Gateway) == nil { + return fmt.Errorf("ipPool.gateway %q is not a valid IP address", ipPool.Gateway) + } + + for _, ns := range ipPool.Nameservers { + if net.ParseIP(ns) == nil { + return fmt.Errorf("ipPool.nameservers entry %q is not a valid IP address", ns) + } + } + + return nil +} + +// validateIPPoolAddress validates a single address entry which can be: +// - A range: "192.168.1.100-192.168.1.120" +// - A CIDR: "192.168.1.0/24" +// - A single IP: "192.168.1.100". +func validateIPPoolAddress(addr string) error { + // Check if it's a range + if strings.Contains(addr, "-") { + parts := strings.Split(addr, "-") + if len(parts) != 2 { + return fmt.Errorf("invalid range format, expected 'start-end'") + } + startIP := net.ParseIP(strings.TrimSpace(parts[0])) + endIP := net.ParseIP(strings.TrimSpace(parts[1])) + if startIP == nil { + return fmt.Errorf("invalid start IP in range") + } + if endIP == nil { + return fmt.Errorf("invalid end IP in range") + } + // Ensure start <= end (simple byte comparison for IPv4) + if bytes.Compare(startIP.To4(), endIP.To4()) > 0 { + return fmt.Errorf("start IP must be less than or equal to end IP") + } + return nil + } + + // Check if it's a CIDR + if strings.Contains(addr, "/") { + _, _, err := net.ParseCIDR(addr) + if err != nil { + return fmt.Errorf("invalid CIDR: %v", err) + } + return nil + } + + // Must be a single IP + if net.ParseIP(addr) == nil { + return fmt.Errorf("invalid IP address") + } + + return nil +} + +// CalculateIPPoolSize calculates the total number of IPs in the pool. +func CalculateIPPoolSize(addresses []string) (int, error) { + total := 0 + for _, addr := range addresses { + // Range + if strings.Contains(addr, "-") { + parts := strings.Split(addr, "-") + startIP := net.ParseIP(strings.TrimSpace(parts[0])).To4() + endIP := net.ParseIP(strings.TrimSpace(parts[1])).To4() + if startIP == nil || endIP == nil { + return 0, fmt.Errorf("invalid IP range: %s", addr) + } + // Calculate range size + start := ipToUint32(startIP) + end := ipToUint32(endIP) + total += int(end - start + 1) + continue + } + + // CIDR + if strings.Contains(addr, "/") { + _, ipNet, err := net.ParseCIDR(addr) + if err != nil { + return 0, fmt.Errorf("invalid CIDR: %s", addr) + } + ones, bits := ipNet.Mask.Size() + // Number of hosts = 2^(bits-ones) - 2 (excluding network and broadcast) + // But for IP pools, we typically include all addresses + total += 1 << (bits - ones) + continue + } + + // Single IP + total++ + } + return total, nil +} + +// ipToUint32 converts an IPv4 address to uint32. +func ipToUint32(ip net.IP) uint32 { + ip = ip.To4() + return uint32(ip[0])<<24 | uint32(ip[1])<<16 | uint32(ip[2])<<8 | uint32(ip[3]) +} diff --git a/pkg/api/v1alpha1/cluster_test.go b/pkg/api/v1alpha1/cluster_test.go index 1caa4d247ceb..7e4487cf19b5 100644 --- a/pkg/api/v1alpha1/cluster_test.go +++ b/pkg/api/v1alpha1/cluster_test.go @@ -4269,3 +4269,182 @@ rules: }) } } + +func TestIPPoolConfigurationEqual(t *testing.T) { + tests := []struct { + name string + pool1 *IPPoolConfiguration + pool2 *IPPoolConfiguration + expect bool + }{ + { + name: "both nil", + pool1: nil, + pool2: nil, + expect: true, + }, + { + name: "first nil", + pool1: nil, + pool2: &IPPoolConfiguration{ + Name: "test", + }, + expect: false, + }, + { + name: "second nil", + pool1: &IPPoolConfiguration{ + Name: "test", + }, + pool2: nil, + expect: false, + }, + { + name: "equal configs", + pool1: &IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8", "8.8.4.4"}, + }, + pool2: &IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8", "8.8.4.4"}, + }, + expect: true, + }, + { + name: "different name", + pool1: &IPPoolConfiguration{ + Name: "test-pool-1", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + }, + pool2: &IPPoolConfiguration{ + Name: "test-pool-2", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + }, + expect: false, + }, + { + name: "different addresses", + pool1: &IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + }, + pool2: &IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.110"}, + Prefix: 24, + Gateway: "192.168.1.1", + }, + expect: false, + }, + { + name: "different prefix", + pool1: &IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + }, + pool2: &IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 16, + Gateway: "192.168.1.1", + }, + expect: false, + }, + { + name: "different gateway", + pool1: &IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + }, + pool2: &IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.254", + }, + expect: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.pool1.Equal(tt.pool2) + assert.Equal(t, tt.expect, result) + }) + } +} + +func TestCalculateIPPoolSize(t *testing.T) { + tests := []struct { + name string + addresses []string + expected int + wantErr bool + }{ + { + name: "single IP", + addresses: []string{"192.168.1.100"}, + expected: 1, + wantErr: false, + }, + { + name: "multiple single IPs", + addresses: []string{"192.168.1.100", "192.168.1.101", "192.168.1.102"}, + expected: 3, + wantErr: false, + }, + { + name: "IP range", + addresses: []string{"192.168.1.100-192.168.1.110"}, + expected: 11, + wantErr: false, + }, + { + name: "CIDR /24", + addresses: []string{"192.168.1.0/24"}, + expected: 256, + wantErr: false, + }, + { + name: "CIDR /28", + addresses: []string{"192.168.1.0/28"}, + expected: 16, + wantErr: false, + }, + { + name: "mixed addresses", + addresses: []string{"192.168.1.100", "192.168.1.110-192.168.1.120", "192.168.2.0/28"}, + expected: 1 + 11 + 16, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := CalculateIPPoolSize(tt.addresses) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + } + }) + } +} diff --git a/pkg/api/v1alpha1/cluster_types.go b/pkg/api/v1alpha1/cluster_types.go index f6fa502a8fbc..d85f88b080b8 100644 --- a/pkg/api/v1alpha1/cluster_types.go +++ b/pkg/api/v1alpha1/cluster_types.go @@ -1332,6 +1332,51 @@ func (n *PodIAMConfig) Equal(o *PodIAMConfig) bool { return n.ServiceAccountIssuer == o.ServiceAccountIssuer } +// IPPoolConfiguration defines the IP pool configuration for static IP assignment. +// When specified, eksctl creates an InClusterIPPool resource from this configuration +// and nodes will be assigned static IPs from this pool via CAPI IPAM. +type IPPoolConfiguration struct { + // Name is the name for the generated InClusterIPPool resource. + // +kubebuilder:validation:Required + Name string `json:"name"` + + // Addresses defines the IP addresses to include in the pool. + // Supports ranges (e.g., "192.168.1.100-192.168.1.120"), + // CIDR blocks (e.g., "192.168.1.0/24"), or individual IPs. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinItems=1 + Addresses []string `json:"addresses"` + + // Prefix is the subnet prefix length (e.g., 24 for /24 subnet). + // +kubebuilder:validation:Required + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=32 + Prefix int `json:"prefix"` + + // Gateway is the default gateway IP address for the subnet. + // +kubebuilder:validation:Required + Gateway string `json:"gateway"` + + // Nameservers is a list of DNS server IP addresses. + // +kubebuilder:validation:Optional + Nameservers []string `json:"nameservers,omitempty"` +} + +// Equal compares two IPPoolConfigurations for equality. +func (n *IPPoolConfiguration) Equal(o *IPPoolConfiguration) bool { + if n == o { + return true + } + if n == nil || o == nil { + return false + } + return n.Name == o.Name && + SliceEqual(n.Addresses, o.Addresses) && + n.Prefix == o.Prefix && + n.Gateway == o.Gateway && + SliceEqual(n.Nameservers, o.Nameservers) +} + // AutoScalingConfiguration defines the configuration for the node autoscaling feature. type AutoScalingConfiguration struct { // MinCount defines the minimum number of nodes for the associated resource group. diff --git a/pkg/api/v1alpha1/vspheredatacenterconfig_types.go b/pkg/api/v1alpha1/vspheredatacenterconfig_types.go index 83196f91949b..2990c7d28983 100644 --- a/pkg/api/v1alpha1/vspheredatacenterconfig_types.go +++ b/pkg/api/v1alpha1/vspheredatacenterconfig_types.go @@ -21,6 +21,12 @@ type VSphereDatacenterConfigSpec struct { Thumbprint string `json:"thumbprint"` Insecure bool `json:"insecure"` FailureDomains []FailureDomain `json:"failureDomains,omitempty"` + + // IPPool defines the IP pool configuration for static IP assignment to cluster nodes. + // When specified, nodes will be assigned static IPs from this pool instead of using DHCP. + // The CLI creates an InClusterIPPool resource from this configuration. + // +kubebuilder:validation:Optional + IPPool *IPPoolConfiguration `json:"ipPool,omitempty"` } // FailureDomain defines the list of failure domains to spread the VMs across. @@ -173,6 +179,13 @@ func (v *VSphereDatacenterConfig) Validate() error { } } + // Validate IPPool configuration if present + if v.Spec.IPPool != nil { + if err := validateIPPoolConfig(v.Spec.IPPool); err != nil { + return fmt.Errorf("invalid ipPool configuration: %v", err) + } + } + return nil } diff --git a/pkg/api/v1alpha1/vspheredatacenterconfig_webhook.go b/pkg/api/v1alpha1/vspheredatacenterconfig_webhook.go index 0e05e706c981..81d732b5af69 100644 --- a/pkg/api/v1alpha1/vspheredatacenterconfig_webhook.go +++ b/pkg/api/v1alpha1/vspheredatacenterconfig_webhook.go @@ -152,6 +152,9 @@ func validateImmutableFieldsVSphereCluster(new, old *VSphereDatacenterConfig) fi ) } + // IPPool configuration is mutable - users can switch between DHCP and static IP modes + // When switching modes, a rolling update of all nodes will be triggered + return allErrs } diff --git a/pkg/api/v1alpha1/vspheredatacenterconfig_webhook_test.go b/pkg/api/v1alpha1/vspheredatacenterconfig_webhook_test.go index e5497753f736..7b495029baeb 100644 --- a/pkg/api/v1alpha1/vspheredatacenterconfig_webhook_test.go +++ b/pkg/api/v1alpha1/vspheredatacenterconfig_webhook_test.go @@ -245,3 +245,146 @@ func TestVSphereDatacenterConfigValidateDeleteCastFail(t *testing.T) { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("expected a VSphereDatacenterConfig")) } + +func TestVSphereDatacenterValidateUpdateIPPoolAddAllowed(t *testing.T) { + ctx := context.Background() + g := NewWithT(t) + + // Old config without IPPool (DHCP mode) + vOld := vsphereDatacenterConfig() + vOld.Spec.IPPool = nil + + // New config with IPPool (static IP mode) + c := vOld.DeepCopy() + c.Spec.IPPool = &v1alpha1.IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + } + + // Should succeed - switching from DHCP to static IP is allowed (triggers rolling update) + g.Expect(c.ValidateUpdate(ctx, &vOld, c)).Error().To(Succeed()) +} + +func TestVSphereDatacenterValidateUpdateIPPoolRemoveAllowed(t *testing.T) { + ctx := context.Background() + g := NewWithT(t) + + // Old config with IPPool (static IP mode) + vOld := vsphereDatacenterConfig() + vOld.Spec.IPPool = &v1alpha1.IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + } + + // New config without IPPool (DHCP mode) + c := vOld.DeepCopy() + c.Spec.IPPool = nil + + // Should succeed - switching from static IP to DHCP is allowed (triggers rolling update) + g.Expect(c.ValidateUpdate(ctx, &vOld, c)).Error().To(Succeed()) +} + +func TestVSphereDatacenterValidateUpdateIPPoolFieldsMutable(t *testing.T) { + ctx := context.Background() + g := NewWithT(t) + + // Old config with IPPool + vOld := vsphereDatacenterConfig() + vOld.Spec.IPPool = &v1alpha1.IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + } + + // New config with modified IPPool fields (but still has IPPool) + c := vOld.DeepCopy() + c.Spec.IPPool = &v1alpha1.IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.150"}, // Extended range + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, // Added nameservers + } + + // Should succeed because updating IPPool fields within the same mode is allowed + g.Expect(c.ValidateUpdate(ctx, &vOld, c)).Error().To(Succeed()) +} + +func TestVSphereDatacenterValidateUpdateBothWithoutIPPool(t *testing.T) { + ctx := context.Background() + g := NewWithT(t) + + // Both old and new without IPPool (DHCP mode) + vOld := vsphereDatacenterConfig() + vOld.Spec.IPPool = nil + + c := vOld.DeepCopy() + c.Spec.IPPool = nil + + // Should succeed because both are using DHCP mode + g.Expect(c.ValidateUpdate(ctx, &vOld, c)).Error().To(Succeed()) +} + +func TestVSphereDatacenterValidateUpdateBothWithIPPool(t *testing.T) { + ctx := context.Background() + g := NewWithT(t) + + // Both old and new with IPPool (static IP mode) + vOld := vsphereDatacenterConfig() + vOld.Spec.IPPool = &v1alpha1.IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + } + + c := vOld.DeepCopy() + // Same IPPool config + c.Spec.IPPool = &v1alpha1.IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + } + + // Should succeed because both are using static IP mode + g.Expect(c.ValidateUpdate(ctx, &vOld, c)).Error().To(Succeed()) +} + +func TestVSphereDatacenterValidateCreateWithIPPool(t *testing.T) { + ctx := context.Background() + g := NewWithT(t) + + dataCenterConfig := vsphereDatacenterConfig() + dataCenterConfig.Spec.IPPool = &v1alpha1.IPPoolConfiguration{ + Name: "test-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + // Should succeed with valid IPPool configuration + g.Expect(dataCenterConfig.ValidateCreate(ctx, &dataCenterConfig)).Error().To(Succeed()) +} + +func TestVSphereDatacenterValidateCreateWithInvalidIPPool(t *testing.T) { + ctx := context.Background() + g := NewWithT(t) + + dataCenterConfig := vsphereDatacenterConfig() + dataCenterConfig.Spec.IPPool = &v1alpha1.IPPoolConfiguration{ + Name: "", // Empty name is invalid + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + } + + // Should fail with invalid IPPool configuration + g.Expect(dataCenterConfig.ValidateCreate(ctx, &dataCenterConfig)).Error().To(MatchError(ContainSubstring("ipPool.name cannot be empty"))) +} diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 8fcff212a374..fff9641b4be5 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -1603,6 +1603,31 @@ func (in *IPPool) DeepCopy() *IPPool { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolConfiguration) DeepCopyInto(out *IPPoolConfiguration) { + *out = *in + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Nameservers != nil { + in, out := &in.Nameservers, &out.Nameservers + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolConfiguration. +func (in *IPPoolConfiguration) DeepCopy() *IPPoolConfiguration { + if in == nil { + return nil + } + out := new(IPPoolConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageResource) DeepCopyInto(out *ImageResource) { *out = *in @@ -3411,6 +3436,11 @@ func (in *VSphereDatacenterConfigSpec) DeepCopyInto(out *VSphereDatacenterConfig *out = make([]FailureDomain, len(*in)) copy(*out, *in) } + if in.IPPool != nil { + in, out := &in.IPPool, &out.IPPool + *out = new(IPPoolConfiguration) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereDatacenterConfigSpec. diff --git a/pkg/providers/vsphere/config/ipam-provider-crds.yaml b/pkg/providers/vsphere/config/ipam-provider-crds.yaml new file mode 100644 index 000000000000..863f159c85df --- /dev/null +++ b/pkg/providers/vsphere/config/ipam-provider-crds.yaml @@ -0,0 +1,123 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: capi-ipam-provider + labels: + cluster.x-k8s.io/provider: ipam-in-cluster +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: inclusterippools.ipam.cluster.x-k8s.io + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + cluster.x-k8s.io/v1beta1: v1beta1 +spec: + group: ipam.cluster.x-k8s.io + names: + kind: InClusterIPPool + listKind: InClusterIPPoolList + plural: inclusterippools + singular: inclusterippool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.addresses + name: Addresses + type: string + - jsonPath: .spec.prefix + name: Prefix + type: integer + - jsonPath: .spec.gateway + name: Gateway + type: string + name: v1alpha2 + schema: + openAPIV3Schema: + description: InClusterIPPool is the Schema for the inclusterippools API. + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + description: InClusterIPPoolSpec defines the desired state of InClusterIPPool. + properties: + addresses: + description: Addresses is a list of IP addresses or CIDR blocks. + items: + type: string + type: array + gateway: + description: Gateway is the gateway IP address. + type: string + prefix: + description: Prefix is the subnet prefix. + type: integer + required: + - addresses + - prefix + - gateway + type: object + status: + description: InClusterIPPoolStatus defines the observed state of InClusterIPPool. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalinclusterippools.ipam.cluster.x-k8s.io + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + cluster.x-k8s.io/v1beta1: v1beta1 +spec: + group: ipam.cluster.x-k8s.io + names: + kind: GlobalInClusterIPPool + listKind: GlobalInClusterIPPoolList + plural: globalinclusterippools + singular: globalinclusterippool + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: GlobalInClusterIPPool is the Schema for the globalinclusterippools API. + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + description: InClusterIPPoolSpec defines the desired state of InClusterIPPool. + properties: + addresses: + items: + type: string + type: array + gateway: + type: string + prefix: + type: integer + required: + - addresses + - prefix + - gateway + type: object + status: + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/providers/vsphere/config/ipam-provider-deployment.yaml b/pkg/providers/vsphere/config/ipam-provider-deployment.yaml new file mode 100644 index 000000000000..9b1b0f72d0bc --- /dev/null +++ b/pkg/providers/vsphere/config/ipam-provider-deployment.yaml @@ -0,0 +1,310 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: caip-in-cluster-controller-manager + namespace: capi-ipam-provider + labels: + cluster.x-k8s.io/provider: ipam-in-cluster +--- +# RBAC for eksa-controller-manager to manage IPAM resources +# This allows the EKS-A controller to create/update InClusterIPPool resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: eksa-ipam-manager-role + labels: + cluster.x-k8s.io/provider: ipam-in-cluster +rules: +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - inclusterippools + - globalinclusterippools + - ipaddressclaims + - ipaddresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: eksa-ipam-manager-rolebinding + labels: + cluster.x-k8s.io/provider: ipam-in-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: eksa-ipam-manager-role +subjects: +- kind: ServiceAccount + name: eksa-controller-manager + namespace: eksa-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: caip-in-cluster-manager-role + labels: + cluster.x-k8s.io/provider: ipam-in-cluster +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + verbs: + - get + - list + - watch +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - globalinclusterippools + verbs: + - get + - list + - watch +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - globalinclusterippools/finalizers + verbs: + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - globalinclusterippools/status + verbs: + - get + - patch + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - inclusterippools + verbs: + - get + - list + - watch +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - inclusterippools/finalizers + verbs: + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - inclusterippools/status + verbs: + - get + - patch + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddressclaims + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddressclaims/finalizers + verbs: + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddressclaims/status + verbs: + - get + - patch + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddresses/finalizers + verbs: + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddresses/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: caip-in-cluster-manager-rolebinding + labels: + cluster.x-k8s.io/provider: ipam-in-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: caip-in-cluster-manager-role +subjects: +- kind: ServiceAccount + name: caip-in-cluster-controller-manager + namespace: capi-ipam-provider +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: caip-in-cluster-leader-election-role + namespace: capi-ipam-provider + labels: + cluster.x-k8s.io/provider: ipam-in-cluster +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: caip-in-cluster-leader-election-rolebinding + namespace: capi-ipam-provider + labels: + cluster.x-k8s.io/provider: ipam-in-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: caip-in-cluster-leader-election-role +subjects: +- kind: ServiceAccount + name: caip-in-cluster-controller-manager + namespace: capi-ipam-provider +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: caip-in-cluster-controller-manager + namespace: capi-ipam-provider + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + control-plane: controller-manager +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: ipam-in-cluster + control-plane: controller-manager + template: + metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + control-plane: controller-manager + spec: + containers: + - name: manager + image: registry.k8s.io/capi-ipam-ic/cluster-api-ipam-in-cluster-controller:v1.0.3 + args: + - --leader-elect + - --metrics-bind-address=:8080 + - --health-probe-bind-address=:8081 + - --webhook-port=0 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + - containerPort: 8081 + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsGroup: 65532 + runAsUser: 65532 + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + serviceAccountName: caip-in-cluster-controller-manager + terminationGracePeriodSeconds: 10 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane diff --git a/pkg/providers/vsphere/config/template-cp.yaml b/pkg/providers/vsphere/config/template-cp.yaml index a1967909b339..2a8e8f2e1ca2 100644 --- a/pkg/providers/vsphere/config/template-cp.yaml +++ b/pkg/providers/vsphere/config/template-cp.yaml @@ -59,8 +59,22 @@ spec: memoryMiB: {{.controlPlaneVMsMemoryMiB}} network: devices: +{{- if .ipPoolName }} + - networkName: {{.vsphereNetwork}} + addressesFromPools: + - apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + name: {{.ipPoolName}} +{{- if .ipPoolNameservers }} + nameservers: +{{- range .ipPoolNameservers }} + - {{ . }} +{{- end }} +{{- end }} +{{- else }} - dhcp4: true networkName: {{.vsphereNetwork}} +{{- end }} numCPUs: {{.controlPlaneVMsNumCPUs}} resourcePool: '{{.controlPlaneVsphereResourcePool}}' server: {{.vsphereServer}} @@ -729,8 +743,22 @@ spec: memoryMiB: {{.etcdVMsMemoryMiB}} network: devices: +{{- if .ipPoolName }} + - networkName: {{.vsphereNetwork}} + addressesFromPools: + - apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + name: {{.ipPoolName}} +{{- if .ipPoolNameservers }} + nameservers: +{{- range .ipPoolNameservers }} + - {{ . }} +{{- end }} +{{- end }} +{{- else }} - dhcp4: true networkName: {{.vsphereNetwork}} +{{- end }} numCPUs: {{.etcdVMsNumCPUs}} resourcePool: '{{.etcdVsphereResourcePool}}' server: {{.vsphereServer}} diff --git a/pkg/providers/vsphere/config/template-ippool.yaml b/pkg/providers/vsphere/config/template-ippool.yaml new file mode 100644 index 000000000000..ec268db51bde --- /dev/null +++ b/pkg/providers/vsphere/config/template-ippool.yaml @@ -0,0 +1,12 @@ +apiVersion: ipam.cluster.x-k8s.io/v1alpha2 +kind: InClusterIPPool +metadata: + name: {{.ipPoolName}} + namespace: {{.ipPoolNamespace}} +spec: + addresses: +{{- range .ipPoolAddresses}} + - {{.}} +{{- end}} + prefix: {{.ipPoolPrefix}} + gateway: {{.ipPoolGateway}} diff --git a/pkg/providers/vsphere/config/template-md.yaml b/pkg/providers/vsphere/config/template-md.yaml index 12bd10873762..9eee5d375b9f 100644 --- a/pkg/providers/vsphere/config/template-md.yaml +++ b/pkg/providers/vsphere/config/template-md.yaml @@ -259,13 +259,41 @@ spec: network: devices: {{- if .vsphereMultiNetworks }} - {{range .vsphereMultiNetworks}} +{{- range $index, $network := .vsphereMultiNetworks}} +{{- if and (eq $index 0) $.ipPoolName }} + - networkName: {{$network}} + addressesFromPools: + - apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + name: {{$.ipPoolName}} +{{- if $.ipPoolNameservers }} + nameservers: +{{- range $.ipPoolNameservers }} + - {{ . }} +{{- end }} +{{- end }} +{{- else }} - dhcp4: true - networkName: {{.}} - {{- end }} + networkName: {{$network}} +{{- end }} +{{- end }} {{- else}} +{{- if .ipPoolName }} + - networkName: {{.vsphereNetwork}} + addressesFromPools: + - apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + name: {{.ipPoolName}} +{{- if .ipPoolNameservers }} + nameservers: +{{- range .ipPoolNameservers }} + - {{ . }} +{{- end }} +{{- end }} +{{- else }} - dhcp4: true networkName: {{.vsphereNetwork}} +{{- end }} {{- end }} numCPUs: {{.workloadVMsNumCPUs}} resourcePool: '{{.workerVsphereResourcePool}}' diff --git a/pkg/providers/vsphere/mocks/client.go b/pkg/providers/vsphere/mocks/client.go index 4a9a5ab9493a..a6a2f486c237 100644 --- a/pkg/providers/vsphere/mocks/client.go +++ b/pkg/providers/vsphere/mocks/client.go @@ -827,6 +827,21 @@ func (mr *MockProviderKubectlClientMockRecorder) GetSecretFromNamespace(arg0, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSecretFromNamespace", reflect.TypeOf((*MockProviderKubectlClient)(nil).GetSecretFromNamespace), arg0, arg1, arg2, arg3) } +// HasCRD mocks base method. +func (m *MockProviderKubectlClient) HasCRD(arg0 context.Context, arg1, arg2 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasCRD", arg0, arg1, arg2) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// HasCRD indicates an expected call of HasCRD. +func (mr *MockProviderKubectlClientMockRecorder) HasCRD(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasCRD", reflect.TypeOf((*MockProviderKubectlClient)(nil).HasCRD), arg0, arg1, arg2) +} + // LoadSecret mocks base method. func (m *MockProviderKubectlClient) LoadSecret(arg0 context.Context, arg1, arg2, arg3, arg4 string) error { m.ctrl.T.Helper() diff --git a/pkg/providers/vsphere/template.go b/pkg/providers/vsphere/template.go index 1c53c64da093..f2126700a802 100644 --- a/pkg/providers/vsphere/template.go +++ b/pkg/providers/vsphere/template.go @@ -445,6 +445,14 @@ func buildTemplateMapCP( } } + // Add IP pool configuration for static IP support via CAPI IPAM + if clusterSpec.VSphereDatacenter.Spec.IPPool != nil { + values["ipPoolName"] = clusterSpec.VSphereDatacenter.Spec.IPPool.Name + if len(clusterSpec.VSphereDatacenter.Spec.IPPool.Nameservers) > 0 { + values["ipPoolNameservers"] = clusterSpec.VSphereDatacenter.Spec.IPPool.Nameservers + } + } + return values, nil } @@ -613,6 +621,14 @@ func buildTemplateMapMD( values["nodeLabelArgs"] = nodeLabelArgs.ToPartialYaml() } + // Add IP pool configuration for static IP support via CAPI IPAM + if clusterSpec.VSphereDatacenter.Spec.IPPool != nil { + values["ipPoolName"] = clusterSpec.VSphereDatacenter.Spec.IPPool.Name + if len(clusterSpec.VSphereDatacenter.Spec.IPPool.Nameservers) > 0 { + values["ipPoolNameservers"] = clusterSpec.VSphereDatacenter.Spec.IPPool.Nameservers + } + } + return values, nil } diff --git a/pkg/providers/vsphere/template_test.go b/pkg/providers/vsphere/template_test.go index 8439e18ccf33..816179a1f430 100644 --- a/pkg/providers/vsphere/template_test.go +++ b/pkg/providers/vsphere/template_test.go @@ -270,3 +270,48 @@ func TestVsphereTemplateBuilderGenerateCAPISpecControlPlaneWithDefaultAuditPolic g.Expect(collapseWhitespace(string(data))).To(ContainSubstring(collapseWhitespace(defaultAuditPolicy))) } + +func TestVsphereTemplateBuilderGenerateCAPISpecControlPlaneWithIPPool(t *testing.T) { + g := NewWithT(t) + spec := test.NewFullClusterSpec(t, "testdata/cluster_main_ippool.yaml") + + builder := vsphere.NewVsphereTemplateBuilder(time.Now) + data, err := builder.GenerateCAPISpecControlPlane(spec, func(values map[string]interface{}) { + values["controlPlaneTemplateName"] = clusterapi.ControlPlaneMachineTemplateName(spec.Cluster) + }) + + g.Expect(err).ToNot(HaveOccurred()) + + // Verify IPPool configuration is present in the generated YAML + yamlStr := string(data) + g.Expect(yamlStr).To(ContainSubstring("addressesFromPools")) + g.Expect(yamlStr).To(ContainSubstring("apiGroup: ipam.cluster.x-k8s.io")) + g.Expect(yamlStr).To(ContainSubstring("kind: InClusterIPPool")) + g.Expect(yamlStr).To(ContainSubstring("name: test-ip-pool")) + g.Expect(yamlStr).To(ContainSubstring("nameservers")) + g.Expect(yamlStr).To(ContainSubstring("8.8.8.8")) + g.Expect(yamlStr).To(ContainSubstring("8.8.4.4")) + // Verify DHCP is NOT present + g.Expect(yamlStr).ToNot(ContainSubstring("dhcp4: true")) +} + +func TestVsphereTemplateBuilderGenerateCAPISpecWorkersWithIPPool(t *testing.T) { + g := NewWithT(t) + spec := test.NewFullClusterSpec(t, "testdata/cluster_main_ippool.yaml") + + builder := vsphere.NewVsphereTemplateBuilder(time.Now) + data, err := builder.GenerateCAPISpecWorkers(spec, nil, nil) + + g.Expect(err).ToNot(HaveOccurred()) + + // Verify IPPool configuration is present in the generated YAML + yamlStr := string(data) + g.Expect(yamlStr).To(ContainSubstring("addressesFromPools")) + g.Expect(yamlStr).To(ContainSubstring("apiGroup: ipam.cluster.x-k8s.io")) + g.Expect(yamlStr).To(ContainSubstring("kind: InClusterIPPool")) + g.Expect(yamlStr).To(ContainSubstring("name: test-ip-pool")) + g.Expect(yamlStr).To(ContainSubstring("nameservers")) + g.Expect(yamlStr).To(ContainSubstring("8.8.8.8")) + // Verify DHCP is NOT present + g.Expect(yamlStr).ToNot(ContainSubstring("dhcp4: true")) +} diff --git a/pkg/providers/vsphere/testdata/cluster_main_ippool.yaml b/pkg/providers/vsphere/testdata/cluster_main_ippool.yaml new file mode 100644 index 000000000000..ce8f4a889b3f --- /dev/null +++ b/pkg/providers/vsphere/testdata/cluster_main_ippool.yaml @@ -0,0 +1,122 @@ +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: Cluster +metadata: + name: test + namespace: test-namespace +spec: + controlPlaneConfiguration: + count: 3 + endpoint: + host: 1.2.3.4 + machineGroupRef: + name: test-cp + kind: VSphereMachineConfig + kubernetesVersion: "1.19" + workerNodeGroupConfigurations: + - count: 3 + machineGroupRef: + name: test-wn + kind: VSphereMachineConfig + name: md-0 + externalEtcdConfiguration: + count: 3 + machineGroupRef: + name: test-etcd + kind: VSphereMachineConfig + datacenterRef: + kind: VSphereDatacenterConfig + name: test + clusterNetwork: + cni: "cilium" + pods: + cidrBlocks: + - 192.168.0.0/16 + services: + cidrBlocks: + - 10.96.0.0/12 + node: + cidrMaskSize: 8 +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: VSphereMachineConfig +metadata: + name: test-cp + namespace: test-namespace +spec: + diskGiB: 25 + cloneMode: linkedClone + datastore: "/SDDC-Datacenter/datastore/WorkloadDatastore" + folder: "/SDDC-Datacenter/vm" + memoryMiB: 8192 + numCPUs: 2 + osFamily: ubuntu + resourcePool: "*/Resources" + storagePolicyName: "vSAN Default Storage Policy" + template: "/SDDC-Datacenter/vm/Templates/ubuntu-1804-kube-v1.19.6" + users: + - name: capv + sshAuthorizedKeys: + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== testemail@test.com" +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: VSphereMachineConfig +metadata: + name: test-wn + namespace: test-namespace +spec: + diskGiB: 25 + cloneMode: linkedClone + datastore: "/SDDC-Datacenter/datastore/WorkloadDatastore" + folder: "/SDDC-Datacenter/vm" + memoryMiB: 4096 + numCPUs: 3 + osFamily: ubuntu + resourcePool: "*/Resources" + storagePolicyName: "vSAN Default Storage Policy" + template: "/SDDC-Datacenter/vm/Templates/ubuntu-1804-kube-v1.19.6" + users: + - name: capv + sshAuthorizedKeys: + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== testemail@test.com" +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: VSphereMachineConfig +metadata: + name: test-etcd + namespace: test-namespace +spec: + diskGiB: 25 + cloneMode: linkedClone + datastore: "/SDDC-Datacenter/datastore/WorkloadDatastore" + folder: "/SDDC-Datacenter/vm" + memoryMiB: 4096 + numCPUs: 3 + osFamily: ubuntu + resourcePool: "*/Resources" + storagePolicyName: "vSAN Default Storage Policy" + template: "/SDDC-Datacenter/vm/Templates/ubuntu-1804-kube-v1.19.6" + users: + - name: capv + sshAuthorizedKeys: + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== testemail@test.com" +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: VSphereDatacenterConfig +metadata: + name: test + namespace: test-namespace +spec: + datacenter: "SDDC-Datacenter" + network: "/SDDC-Datacenter/network/sddc-cgw-network-1" + server: "vsphere_server" + thumbprint: "ABCDEFG" + insecure: false + ipPool: + name: "test-ip-pool" + addresses: + - "192.168.1.100-192.168.1.120" + prefix: 24 + gateway: "192.168.1.1" + nameservers: + - "8.8.8.8" + - "8.8.4.4" diff --git a/pkg/providers/vsphere/testdata/expected_results_networks_md.yaml b/pkg/providers/vsphere/testdata/expected_results_networks_md.yaml index ffc1f0904ab9..f20fa073b859 100644 --- a/pkg/providers/vsphere/testdata/expected_results_networks_md.yaml +++ b/pkg/providers/vsphere/testdata/expected_results_networks_md.yaml @@ -74,7 +74,6 @@ spec: memoryMiB: 4096 network: devices: - - dhcp4: true networkName: /SDDC-Datacenter/network/network-1 - dhcp4: true diff --git a/pkg/providers/vsphere/vsphere.go b/pkg/providers/vsphere/vsphere.go index b63162dfeb14..d4c9f2cbae10 100644 --- a/pkg/providers/vsphere/vsphere.go +++ b/pkg/providers/vsphere/vsphere.go @@ -67,6 +67,15 @@ var defaultSecretObject string //go:embed config/template-failuredomain.yaml var defaultFailureDomainConfig string +//go:embed config/ipam-provider-crds.yaml +var defaultIPAMProviderCRDs string + +//go:embed config/ipam-provider-deployment.yaml +var defaultIPAMProviderDeployment string + +//go:embed config/template-ippool.yaml +var defaultIPPoolTemplate string + var ( eksaVSphereDatacenterResourceType = fmt.Sprintf("vspheredatacenterconfigs.%s", v1alpha1.GroupVersion.Group) eksaVSphereMachineResourceType = fmt.Sprintf("vspheremachineconfigs.%s", v1alpha1.GroupVersion.Group) @@ -75,6 +84,7 @@ var ( var requiredEnvs = []string{vSphereUsernameKey, vSpherePasswordKey, expClusterResourceSetKey} type vsphereProvider struct { + datacenterConfig *v1alpha1.VSphereDatacenterConfig clusterConfig *v1alpha1.Cluster providerGovcClient ProviderGovcClient providerKubectlClient ProviderKubectlClient @@ -149,6 +159,7 @@ type ProviderKubectlClient interface { DeleteEksaDatacenterConfig(ctx context.Context, vsphereDatacenterResourceType, vsphereDatacenterConfigName, kubeconfigFile, namespace string) error DeleteEksaMachineConfig(ctx context.Context, vsphereMachineResourceType, vsphereMachineConfigName, kubeconfigFile, namespace string) error ApplyTolerationsFromTaintsToDaemonSet(ctx context.Context, oldTaints, newTaints []corev1.Taint, dsName, kubeconfigFile string) error + HasCRD(ctx context.Context, crd, kubeconfig string) (bool, error) } // IPValidator is an interface that defines methods to validate the control plane IP. @@ -207,6 +218,7 @@ func NewProviderCustomNet( // We should make it exported, but that would involve a bunch of changes, so will do it separately retrier := retrier.NewWithMaxRetries(maxRetries, backOffPeriod) return &vsphereProvider{ + datacenterConfig: datacenterConfig, clusterConfig: clusterConfig, providerGovcClient: providerGovcClient, providerKubectlClient: providerKubectlClient, @@ -291,6 +303,11 @@ func (p *vsphereProvider) SetupAndValidateCreateCluster(ctx context.Context, clu return err } + // Validate IP pool size if configured + if err := validateIPPoolSize(clusterSpec); err != nil { + return err + } + if err := p.validator.ValidateVCenterConfig(ctx, vSphereClusterSpec.VSphereDatacenter); err != nil { return err } @@ -378,6 +395,11 @@ func (p *vsphereProvider) SetupAndValidateUpgradeCluster(ctx context.Context, cl return err } + // Validate IP pool size if configured + if err := validateIPPoolSize(clusterSpec); err != nil { + return err + } + if err := p.validator.ValidateVCenterConfig(ctx, vSphereClusterSpec.VSphereDatacenter); err != nil { return err } @@ -736,10 +758,191 @@ func (p *vsphereProvider) createSecret(ctx context.Context, cluster *types.Clust } func (p *vsphereProvider) PreCAPIInstallOnBootstrap(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec) error { - return p.UpdateSecrets(ctx, cluster, nil) + if err := p.UpdateSecrets(ctx, cluster, nil); err != nil { + return err + } + + // Install IPAM provider and create InClusterIPPool BEFORE CAPI installs VSphereMachineTemplates. + // This is critical because VSphereMachineTemplates reference the InClusterIPPool via addressesFromPools, + // and CAPV will fail to create IPAddressClaims if the pool doesn't exist. + if clusterSpec != nil && clusterSpec.Config != nil && clusterSpec.VSphereDatacenter != nil && clusterSpec.VSphereDatacenter.Spec.IPPool != nil { + logger.V(3).Info("Installing IPAM provider for static IP allocation", + "poolName", clusterSpec.VSphereDatacenter.Spec.IPPool.Name, + ) + if err := p.installIPAMProviderAndCreatePool(ctx, cluster, clusterSpec.VSphereDatacenter); err != nil { + return fmt.Errorf("setting up static IP allocation in PreCAPIInstallOnBootstrap: %v", err) + } + } + + return nil } func (p *vsphereProvider) PostBootstrapSetup(ctx context.Context, clusterConfig *v1alpha1.Cluster, cluster *types.Cluster) error { + // IPAM setup has been moved to PreCAPIInstallOnBootstrap to ensure the InClusterIPPool + // exists BEFORE VSphereMachineTemplates are applied. This is required because CAPV + // creates IPAddressClaims when processing VSphereMachineTemplates with addressesFromPools. + return nil +} + +// installIPAMProviderAndCreatePool installs the CAPI IPAM provider and creates the InClusterIPPool. +func (p *vsphereProvider) installIPAMProviderAndCreatePool(ctx context.Context, cluster *types.Cluster, datacenterConfig *v1alpha1.VSphereDatacenterConfig) error { + // Step 1: Install CAPI IPAM provider (cluster-api-ipam-provider-in-cluster) + if err := p.installCAPIIPAMProvider(ctx, cluster); err != nil { + return fmt.Errorf("installing CAPI IPAM provider: %v", err) + } + + // Step 2: Create InClusterIPPool resource + if err := p.createInClusterIPPoolFromConfig(ctx, cluster, datacenterConfig); err != nil { + return fmt.Errorf("creating InClusterIPPool: %v", err) + } + + return nil +} + +// installCAPIIPAMProvider installs the cluster-api-ipam-provider-in-cluster components. +// Note: IPAddressClaim and IPAddress CRDs are part of core CAPI (v1.5+) and are already installed. +// This function only installs the InClusterIPPool/GlobalInClusterIPPool CRDs and the IPAM controller. +// The installation is split into two phases to avoid controller startup failures: +// 1. Apply CRDs and namespace first, wait for CRDs to be established +// 2. Apply RBAC and deployment after CRDs are ready. +func (p *vsphereProvider) installCAPIIPAMProvider(ctx context.Context, cluster *types.Cluster) error { + // Step 1: Apply CRDs and namespace first + err := p.providerKubectlClient.ApplyKubeSpecFromBytes(ctx, cluster, []byte(defaultIPAMProviderCRDs)) + if err != nil { + return fmt.Errorf("applying IPAM provider CRDs: %v", err) + } + + // Step 2: Wait for CRDs to be established before deploying the controller + // This prevents the controller from failing on startup due to missing API resources + if err := p.waitForIPAMCRDs(ctx, cluster); err != nil { + return fmt.Errorf("waiting for IPAM CRDs to be established: %v", err) + } + + // Step 3: Apply RBAC and deployment + err = p.providerKubectlClient.ApplyKubeSpecFromBytes(ctx, cluster, []byte(defaultIPAMProviderDeployment)) + if err != nil { + return fmt.Errorf("applying IPAM provider deployment: %v", err) + } + + logger.V(3).Info("CAPI IPAM provider installed successfully") + return nil +} + +// waitForIPAMCRDs waits for the IPAM CRDs to be established in the cluster. +func (p *vsphereProvider) waitForIPAMCRDs(ctx context.Context, cluster *types.Cluster) error { + crdsToWait := []string{ + "inclusterippools.ipam.cluster.x-k8s.io", + "globalinclusterippools.ipam.cluster.x-k8s.io", + } + + for _, crdName := range crdsToWait { + err := p.Retrier.Retry(func() error { + hasCRD, err := p.providerKubectlClient.HasCRD(ctx, crdName, cluster.KubeconfigFile) + if err != nil { + return err + } + if !hasCRD { + return fmt.Errorf("CRD %s not yet established", crdName) + } + return nil + }) + if err != nil { + return fmt.Errorf("waiting for CRD %s: %v", crdName, err) + } + } + + return nil +} + +// validateIPPoolSize validates that the IP pool has sufficient IPs for the cluster nodes. +// It calculates the total number of nodes (control plane + workers + etcd) and ensures +// the pool has enough addresses, including extra IPs for rolling upgrades. +// For worker nodes with autoscaling, it uses MaxCount to ensure the pool can accommodate scale-up. +func validateIPPoolSize(clusterSpec *cluster.Spec) error { + if clusterSpec.VSphereDatacenter == nil || clusterSpec.VSphereDatacenter.Spec.IPPool == nil { + return nil + } + + ipPool := clusterSpec.VSphereDatacenter.Spec.IPPool + clusterConfig := clusterSpec.Cluster + + // Calculate total nodes needed + cpCount := clusterConfig.Spec.ControlPlaneConfiguration.Count + workerCount := 0 + etcdCount := 0 + + for _, wng := range clusterConfig.Spec.WorkerNodeGroupConfigurations { + // For autoscaling, use MaxCount to ensure pool has enough IPs for scale-up + if wng.AutoScalingConfiguration != nil && wng.AutoScalingConfiguration.MaxCount > 0 { + workerCount += wng.AutoScalingConfiguration.MaxCount + } else if wng.Count != nil { + workerCount += *wng.Count + } + } + if clusterConfig.Spec.ExternalEtcdConfiguration != nil { + etcdCount = clusterConfig.Spec.ExternalEtcdConfiguration.Count + } + + totalNodes := cpCount + workerCount + etcdCount + + // Calculate pool size + poolSize, err := v1alpha1.CalculateIPPoolSize(ipPool.Addresses) + if err != nil { + return fmt.Errorf("failed to calculate IP pool size: %v", err) + } + + // Need extra IPs for rolling upgrades (maxSurge) + requiredIPs := totalNodes + 1 // +1 for rolling upgrade buffer + if poolSize < requiredIPs { + return fmt.Errorf("ipPool '%s' has %d addresses but cluster requires at least %d (control plane: %d, workers: %d, etcd: %d, rolling upgrade buffer: 1)", + ipPool.Name, poolSize, requiredIPs, cpCount, workerCount, etcdCount) + } + + logger.Info("IP pool size validation passed", + "poolName", ipPool.Name, + "poolSize", poolSize, + "totalNodes", totalNodes, + "requiredIPs", requiredIPs, + ) + + return nil +} + +// createInClusterIPPoolFromConfig creates an InClusterIPPool resource from the datacenter config. +func (p *vsphereProvider) createInClusterIPPoolFromConfig(ctx context.Context, cluster *types.Cluster, datacenterConfig *v1alpha1.VSphereDatacenterConfig) error { + ipPool := datacenterConfig.Spec.IPPool + if ipPool == nil { + return nil + } + + // Build the InClusterIPPool YAML using template + values := map[string]interface{}{ + "ipPoolName": ipPool.Name, + "ipPoolNamespace": constants.EksaSystemNamespace, + "ipPoolAddresses": ipPool.Addresses, + "ipPoolPrefix": ipPool.Prefix, + "ipPoolGateway": ipPool.Gateway, + } + + t, err := template.New("ippool").Parse(defaultIPPoolTemplate) + if err != nil { + return fmt.Errorf("parsing InClusterIPPool template: %v", err) + } + + var poolYAML bytes.Buffer + if err := t.Execute(&poolYAML, values); err != nil { + return fmt.Errorf("executing InClusterIPPool template: %v", err) + } + + err = p.providerKubectlClient.ApplyKubeSpecFromBytes(ctx, cluster, poolYAML.Bytes()) + if err != nil { + return fmt.Errorf("applying InClusterIPPool: %v", err) + } + + logger.V(3).Info("InClusterIPPool created successfully", + "name", ipPool.Name, + "namespace", constants.EksaSystemNamespace, + ) return nil } @@ -960,6 +1163,28 @@ func machineRefSliceToMap(machineRefs []v1alpha1.Ref) map[string]v1alpha1.Ref { } func (p *vsphereProvider) InstallCustomProviderComponents(ctx context.Context, kubeconfigFile string) error { + // Check if IPPool is configured in VSphereDatacenterConfig - if so, install IPAM provider on the management cluster + if p.datacenterConfig != nil && p.datacenterConfig.Spec.IPPool != nil { + logger.V(3).Info("Installing IPAM provider on management cluster", + "poolName", p.datacenterConfig.Spec.IPPool.Name, + ) + + // Create a temporary cluster object with the kubeconfig for kubectl operations + targetCluster := &types.Cluster{ + KubeconfigFile: kubeconfigFile, + } + + // Step 1: Install CAPI IPAM provider (CRDs, RBAC, Deployment) + if err := p.installCAPIIPAMProvider(ctx, targetCluster); err != nil { + return fmt.Errorf("installing CAPI IPAM provider on management cluster: %v", err) + } + + // Step 2: Create InClusterIPPool resource + if err := p.createInClusterIPPoolFromConfig(ctx, targetCluster, p.datacenterConfig); err != nil { + return fmt.Errorf("creating InClusterIPPool on management cluster: %v", err) + } + } + return nil } diff --git a/pkg/workflows/management/create_move_capi.go b/pkg/workflows/management/create_move_capi.go index dd77168deca4..f12fe5f9e53c 100644 --- a/pkg/workflows/management/create_move_capi.go +++ b/pkg/workflows/management/create_move_capi.go @@ -18,6 +18,15 @@ func (s *moveClusterManagementTask) Run(ctx context.Context, commandContext *tas return &workflows.CollectDiagnosticsTask{} } + // Install custom provider components (like IPAM provider) on the management cluster BEFORE the move + // This is required because after clusterctl move, the management cluster needs the IPAM provider + // to reconcile the cluster resources and update controlPlaneInitialized status + logger.Info("Installing custom provider components on management cluster before move") + if err := commandContext.Provider.InstallCustomProviderComponents(ctx, commandContext.WorkloadCluster.KubeconfigFile); err != nil { + commandContext.SetError(err) + return &workflows.CollectDiagnosticsTask{} + } + logger.Info("Moving the cluster management components from the bootstrap cluster to the management cluster") err = commandContext.ClusterManager.MoveCAPI(ctx, commandContext.BootstrapCluster, commandContext.WorkloadCluster, commandContext.WorkloadCluster.Name, commandContext.ClusterSpec, types.WithNodeRef()) if err != nil { diff --git a/pkg/workflows/management/create_test.go b/pkg/workflows/management/create_test.go index 2ccb3e6f9fea..8d01ef615d86 100644 --- a/pkg/workflows/management/create_test.go +++ b/pkg/workflows/management/create_test.go @@ -219,8 +219,16 @@ func (c *createTestSetup) expectPauseReconcile(err error) { } func (c *createTestSetup) expectMoveManagement(err error) { - c.clusterManager.EXPECT().MoveCAPI( - c.ctx, c.bootstrapCluster, c.workloadCluster, c.workloadCluster.Name, c.clusterSpec, gomock.Any()).Return(err) + c.expectMoveManagementWithInstallErr(nil, err) +} + +func (c *createTestSetup) expectMoveManagementWithInstallErr(installErr, moveErr error) { + gomock.InOrder( + c.provider.EXPECT().InstallCustomProviderComponents( + c.ctx, c.workloadCluster.KubeconfigFile).Return(installErr), + c.clusterManager.EXPECT().MoveCAPI( + c.ctx, c.bootstrapCluster, c.workloadCluster, c.workloadCluster.Name, c.clusterSpec, gomock.Any()).Return(moveErr).MaxTimes(1), + ) } func (c *createTestSetup) expectInstallEksaComponentsWorkload(err1, err2, err3, err4, err5 error) { @@ -683,6 +691,30 @@ func TestCreateMoveCAPIFailure(t *testing.T) { } } +func TestCreateMoveInstallCustomProviderComponentsFailure(t *testing.T) { + c := newCreateTest(t) + c.expectSetup() + c.expectCreateBootstrap() + c.expectPreflightValidationsToPass() + c.expectCAPIInstall(nil, nil, nil) + c.expectInstallEksaComponentsBootstrap(nil, nil, nil, nil) + c.expectCreateWorkload(nil, nil, nil, nil, nil, nil) + c.expectInstallResourcesOnManagementTask(nil) + c.expectPauseReconcile(nil) + c.expectMoveManagementWithInstallErr(errors.New("failed to install custom provider components"), nil) + c.expectCreateNamespace() + + c.clusterManager.EXPECT().SaveLogsManagementCluster(c.ctx, c.clusterSpec, c.bootstrapCluster) + c.clusterManager.EXPECT().SaveLogsWorkloadCluster(c.ctx, c.provider, c.clusterSpec, c.workloadCluster) + + c.writer.EXPECT().Write(fmt.Sprintf("%s-checkpoint.yaml", c.clusterSpec.Cluster.Name), gomock.Any()) + + err := c.run() + if err == nil { + t.Fatalf("Create.Run() expected to return an error when InstallCustomProviderComponents fails") + } +} + func TestPauseReconcilerFailure(t *testing.T) { c := newCreateTest(t) c.expectSetup() diff --git a/pkg/workflows/management/upgrade_cluster.go b/pkg/workflows/management/upgrade_cluster.go index 27c84b87cba0..a24818329f3a 100644 --- a/pkg/workflows/management/upgrade_cluster.go +++ b/pkg/workflows/management/upgrade_cluster.go @@ -17,6 +17,16 @@ type upgradeCluster struct{} // Run upgradeCluster performs actions needed to upgrade the management cluster. func (s *upgradeCluster) Run(ctx context.Context, commandContext *task.CommandContext) task.Task { logger.Info("Upgrading management cluster") + + // Install custom provider components (e.g., IPAM provider for static IP) before upgrading. + // This ensures that any new provider resources (like InClusterIPPool) are available + // when the upgraded VSphereMachineTemplates reference them. + logger.Info("Installing custom provider components on management cluster") + if err := commandContext.Provider.InstallCustomProviderComponents(ctx, commandContext.ManagementCluster.KubeconfigFile); err != nil { + commandContext.SetError(err) + return &workflows.CollectMgmtClusterDiagnosticsTask{} + } + if commandContext.ClusterSpec.Cluster.Spec.DatacenterRef.Kind == v1alpha1.TinkerbellDatacenterKind { clientutil.AddAnnotation(commandContext.ClusterSpec.TinkerbellDatacenter, v1alpha1.ManagedByCLIAnnotation, "true") } diff --git a/pkg/workflows/management/upgrade_test.go b/pkg/workflows/management/upgrade_test.go index 5e70a1d4e02f..5e502709b791 100644 --- a/pkg/workflows/management/upgrade_test.go +++ b/pkg/workflows/management/upgrade_test.go @@ -259,6 +259,7 @@ func (c *upgradeManagementTestSetup) expectApplyReleases(err error) { func (c *upgradeManagementTestSetup) expectUpgradeManagementCluster() { gomock.InOrder( + c.provider.EXPECT().InstallCustomProviderComponents(c.ctx, c.managementCluster.KubeconfigFile).Return(nil), c.clusterUpgrader.EXPECT().Run(c.ctx, c.newClusterSpec, *c.managementCluster).Return(nil), c.clientFactory.EXPECT().BuildClientFromKubeconfig(c.managementCluster.KubeconfigFile).Return(c.client, nil), ) @@ -587,6 +588,7 @@ func TestUpgradeManagementRunFailedUpgrade(t *testing.T) { test.expectInstallEksdManifest(nil) test.expectApplyBundles(nil) test.expectApplyReleases(nil) + test.provider.EXPECT().InstallCustomProviderComponents(test.ctx, test.managementCluster.KubeconfigFile).Return(nil) test.clusterUpgrader.EXPECT().Run(test.ctx, test.newClusterSpec, *test.managementCluster).Return(errors.New("failed upgrading")) test.expectSaveLogs() test.expectWriteCheckpointFile() @@ -614,6 +616,7 @@ func TestUpgradeManagementRunFailedUpgradeClusterBuildClientFromKubeconfig(t *te test.expectInstallEksdManifest(nil) test.expectApplyBundles(nil) test.expectApplyReleases(nil) + test.provider.EXPECT().InstallCustomProviderComponents(test.ctx, test.managementCluster.KubeconfigFile).Return(nil) test.clusterUpgrader.EXPECT().Run(test.ctx, test.newClusterSpec, *test.managementCluster).Return(errors.New("failed upgrading")) test.expectSaveLogs() test.expectWriteCheckpointFile() diff --git a/test/e2e/staticip.go b/test/e2e/staticip.go new file mode 100644 index 000000000000..6dcdec79e601 --- /dev/null +++ b/test/e2e/staticip.go @@ -0,0 +1,94 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "github.com/aws/eks-anywhere/test/framework" +) + +// runStaticIPSimpleFlow runs a simple cluster creation flow with static IP configuration. +// This test validates: +// - Cluster creation with IP pool configuration +// - InClusterIPPool resource is created +// - IPAddressClaim and IPAddress resources are created by CAPV +// - Nodes receive IPs from the configured pool +// - Cluster deletion cleans up IP resources +func runStaticIPSimpleFlow(test *framework.ClusterE2ETest, ipConfig framework.StaticIPConfig) { + test.GenerateClusterConfig() + test.CreateCluster() + + // Validate CAPI IPAM resources + test.ValidateInClusterIPPool(ipConfig.PoolName) + test.ValidateIPAddressResources(ipConfig.PoolName) + test.ValidateStaticIPAllocation(ipConfig) + + test.DeleteCluster() +} + +// runStaticIPUpgradeFlow runs a cluster upgrade flow with static IP configuration. +// This test validates: +// - Cluster creation with IP pool +// - Cluster upgrade preserves IP pool configuration +// - New nodes during rolling upgrade get IPs from the same pool +// - IPs are properly recycled during upgrade +func runStaticIPUpgradeFlow(test *framework.ClusterE2ETest, ipConfig framework.StaticIPConfig, clusterOpts ...framework.ClusterE2ETestOpt) { + test.GenerateClusterConfig() + test.CreateCluster() + + // Validate initial static IP allocation + test.ValidateInClusterIPPool(ipConfig.PoolName) + test.ValidateStaticIPAllocation(ipConfig) + + // Perform upgrade + test.UpgradeClusterWithNewConfig(clusterOpts) + test.ValidateClusterState() + + // Validate IPs after upgrade + test.ValidateIPAddressResources(ipConfig.PoolName) + test.ValidateStaticIPAllocation(ipConfig) + + test.StopIfFailed() + test.DeleteCluster() +} + +// runStaticIPScaleFlow runs a cluster scaling flow with static IP configuration. +// This test validates: +// - Cluster creation with IP pool +// - Scale up allocates new IPs from the pool +// - Scale down releases IPs back to the pool +func runStaticIPScaleFlow(test *framework.ClusterE2ETest, ipConfig framework.StaticIPConfig, scaleUpOpts, scaleDownOpts []framework.ClusterE2ETestOpt) { + test.GenerateClusterConfig() + test.CreateCluster() + + // Validate initial state + test.ValidateInClusterIPPool(ipConfig.PoolName) + test.ValidateStaticIPAllocation(ipConfig) + + // Scale up + test.UpgradeClusterWithNewConfig(scaleUpOpts) + test.ValidateClusterState() + test.ValidateIPAddressResources(ipConfig.PoolName) + test.ValidateStaticIPAllocation(ipConfig) + + // Scale down + test.UpgradeClusterWithNewConfig(scaleDownOpts) + test.ValidateClusterState() + test.ValidateIPReleasedAfterScaleDown(ipConfig.PoolName, 1) + + test.StopIfFailed() + test.DeleteCluster() +} + +// runStaticIPWithoutClusterConfigGeneration runs the static IP flow using +// pre-configured cluster config (set via WithClusterConfig). +func runStaticIPWithoutClusterConfigGeneration(test *framework.ClusterE2ETest, ipConfig framework.StaticIPConfig) { + test.CreateCluster() + + // Validate CAPI IPAM resources + test.ValidateInClusterIPPool(ipConfig.PoolName) + test.ValidateIPAddressResources(ipConfig.PoolName) + test.ValidateStaticIPAllocation(ipConfig) + + test.DeleteCluster() +} diff --git a/test/e2e/vsphere_test.go b/test/e2e/vsphere_test.go index d874b71a116e..3872c50ec083 100644 --- a/test/e2e/vsphere_test.go +++ b/test/e2e/vsphere_test.go @@ -9160,3 +9160,368 @@ func TestVSphereKubernetes134BottlerocketKubeletConfiguration(t *testing.T) { ) runKubeletConfigurationFlow(test) } + +// Static IP Tests - CAPI IPAM with InClusterIPPool + +// TestVSphereKubernetes130UbuntuStaticIPSimpleFlow tests cluster creation with static IP allocation using CAPI IPAM. +func TestVSphereKubernetes130UbuntuStaticIPSimpleFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8", "8.8.4.4"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithUbuntu130()), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + api.VSphereToConfigFiller( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + ), + ) + runStaticIPWithoutClusterConfigGeneration(test, ipConfig) +} + +// TestVSphereKubernetes130BottlerocketStaticIPSimpleFlow tests static IP allocation with Bottlerocket OS. +func TestVSphereKubernetes130BottlerocketStaticIPSimpleFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-br", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithBottleRocket130()), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + api.VSphereToConfigFiller( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + ), + ) + runStaticIPWithoutClusterConfigGeneration(test, ipConfig) +} + +// TestVSphereKubernetes131UbuntuStaticIPSimpleFlow tests cluster creation with static IP using K8s 1.31. +func TestVSphereKubernetes131UbuntuStaticIPSimpleFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-131", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8", "8.8.4.4"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithUbuntu131()), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithKubernetesVersion(v1alpha1.Kube131), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + api.VSphereToConfigFiller( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + ), + ) + runStaticIPWithoutClusterConfigGeneration(test, ipConfig) +} + +// TestVSphereKubernetes130UbuntuStaticIPUpgradeFlow tests cluster upgrade with static IP configuration. +func TestVSphereKubernetes130UbuntuStaticIPUpgradeFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-upgrade", + Addresses: []string{"192.168.1.100-192.168.1.130"}, // Extra IPs for rolling upgrade + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithUbuntu130(), framework.WithVSphereFillers( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + )), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + ) + + runStaticIPUpgradeFlow(test, ipConfig, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube131), + ), + ) +} + +// TestVSphereKubernetes130UbuntuStaticIPScaleFlow tests scaling with static IP allocation. +func TestVSphereKubernetes130UbuntuStaticIPScaleFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-scale", + Addresses: []string{"192.168.1.100-192.168.1.130"}, // Extra IPs for scaling + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithUbuntu130(), framework.WithVSphereFillers( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + )), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + ) + + runStaticIPScaleFlow(test, ipConfig, + // Scale up to 4 workers + []framework.ClusterE2ETestOpt{ + framework.WithClusterUpgrade(api.WithWorkerNodeCount(4)), + }, + // Scale down to 2 workers + []framework.ClusterE2ETestOpt{ + framework.WithClusterUpgrade(api.WithWorkerNodeCount(2)), + }, + ) +} + +// TestVSphereKubernetes130UbuntuStaticIPStackedEtcd tests static IP with stacked etcd topology. +func TestVSphereKubernetes130UbuntuStaticIPStackedEtcd(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-stacked", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithUbuntu130()), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(3), + api.WithWorkerNodeCount(2), + api.WithStackedEtcdTopology(), + ), + api.VSphereToConfigFiller( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + ), + ) + runStaticIPWithoutClusterConfigGeneration(test, ipConfig) +} + +// TestVSphereKubernetes130UbuntuStaticIPCIDRFormat tests static IP with CIDR address format. +func TestVSphereKubernetes130UbuntuStaticIPCIDRFormat(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-cidr", + Addresses: []string{"192.168.1.0/28"}, // 16 IPs (192.168.1.0 - 192.168.1.15) + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithUbuntu130()), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + api.VSphereToConfigFiller( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + ), + ) + runStaticIPWithoutClusterConfigGeneration(test, ipConfig) +} + +// RedHat Static IP Tests + +// TestVSphereKubernetes130RedHat8StaticIPSimpleFlow tests static IP allocation with RedHat 8 OS. +func TestVSphereKubernetes130RedHat8StaticIPSimpleFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-rh8", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8", "8.8.4.4"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithRedHat130VSphere()), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + api.VSphereToConfigFiller( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + ), + ) + runStaticIPWithoutClusterConfigGeneration(test, ipConfig) +} + +// TestVSphereKubernetes131RedHat8StaticIPSimpleFlow tests static IP allocation with RedHat 8 OS on K8s 1.31. +func TestVSphereKubernetes131RedHat8StaticIPSimpleFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-rh8-131", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithRedHat131VSphere()), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithKubernetesVersion(v1alpha1.Kube131), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + api.VSphereToConfigFiller( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + ), + ) + runStaticIPWithoutClusterConfigGeneration(test, ipConfig) +} + +// TestVSphereKubernetes130RedHat9StaticIPSimpleFlow tests static IP allocation with RedHat 9 OS. +func TestVSphereKubernetes130RedHat9StaticIPSimpleFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-rh9", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8", "8.8.4.4"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithRedHat9130VSphere()), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + api.VSphereToConfigFiller( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + ), + ) + runStaticIPWithoutClusterConfigGeneration(test, ipConfig) +} + +// TestVSphereKubernetes131RedHat9StaticIPSimpleFlow tests static IP allocation with RedHat 9 OS on K8s 1.31. +func TestVSphereKubernetes131RedHat9StaticIPSimpleFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-rh9-131", + Addresses: []string{"192.168.1.100-192.168.1.120"}, + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithRedHat9131VSphere()), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithKubernetesVersion(v1alpha1.Kube131), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + api.VSphereToConfigFiller( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + ), + ) + runStaticIPWithoutClusterConfigGeneration(test, ipConfig) +} + +// TestVSphereKubernetes130RedHat9StaticIPUpgradeFlow tests cluster upgrade with RedHat 9 and static IP. +func TestVSphereKubernetes130RedHat9StaticIPUpgradeFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-rh9-upgrade", + Addresses: []string{"192.168.1.100-192.168.1.130"}, // Extra IPs for rolling upgrade + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithRedHat9130VSphere(), framework.WithVSphereFillers( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + )), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + ) + + runStaticIPUpgradeFlow(test, ipConfig, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube131), + ), + ) +} + +// TestVSphereKubernetes130RedHat9StaticIPScaleFlow tests scaling with RedHat 9 and static IP allocation. +func TestVSphereKubernetes130RedHat9StaticIPScaleFlow(t *testing.T) { + ipConfig := framework.StaticIPConfig{ + PoolName: "test-ip-pool-rh9-scale", + Addresses: []string{"192.168.1.100-192.168.1.130"}, // Extra IPs for scaling + Prefix: 24, + Gateway: "192.168.1.1", + Nameservers: []string{"8.8.8.8"}, + } + + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithRedHat9130VSphere(), framework.WithVSphereFillers( + api.WithVSphereIPPool(ipConfig.PoolName, ipConfig.Addresses, ipConfig.Prefix, ipConfig.Gateway, ipConfig.Nameservers), + )), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube130), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(2), + ), + ) + + runStaticIPScaleFlow(test, ipConfig, + // Scale up to 4 workers + []framework.ClusterE2ETestOpt{ + framework.WithClusterUpgrade(api.WithWorkerNodeCount(4)), + }, + // Scale down to 2 workers + []framework.ClusterE2ETestOpt{ + framework.WithClusterUpgrade(api.WithWorkerNodeCount(2)), + }, + ) +} diff --git a/test/framework/staticip.go b/test/framework/staticip.go new file mode 100644 index 000000000000..419097083b6d --- /dev/null +++ b/test/framework/staticip.go @@ -0,0 +1,358 @@ +package framework + +import ( + "context" + "encoding/json" + "fmt" + "net" + "strings" + "time" +) + +// StaticIPConfig holds the configuration for static IP E2E tests. +type StaticIPConfig struct { + PoolName string + Addresses []string + Prefix int + Gateway string + Nameservers []string +} + +// ValidateStaticIPAllocation validates that nodes received static IPs from the configured IP pool. +func (e *ClusterE2ETest) ValidateStaticIPAllocation(config StaticIPConfig) { + e.T.Log("Validating static IP allocation from IP pool") + + // Get all machines + machines, err := e.getAllMachines() + if err != nil { + e.T.Fatalf("Failed to get machines: %v", err) + } + + // Parse expected IP range + expectedIPs := parseIPRange(config.Addresses) + + for _, machine := range machines { + e.T.Logf("Checking machine %s for static IP allocation", machine.Name) + + // Wait for machine to have an IP + err = e.waitForMachineIP(machine.Name, "5m") + if err != nil { + e.T.Fatalf("Machine %s failed to get IP: %v", machine.Name, err) + } + + // Get the external IP + externalIPs := e.getExternalIPsFromMachine(machine) + if len(externalIPs) == 0 { + e.T.Fatalf("Machine %s has no external IPs", machine.Name) + } + + // Verify IP is in the expected range + machineIP := externalIPs[0] + if !isIPInRange(machineIP, expectedIPs) { + e.T.Fatalf("Machine %s IP %s is not in the expected IP pool range %v", + machine.Name, machineIP, config.Addresses) + } + + e.T.Logf("Machine %s has valid static IP %s from pool ✓", machine.Name, machineIP) + } + + e.T.Log("Static IP allocation validation completed successfully") +} + +// ipAddressClaim represents the structure of an IPAddressClaim resource. +type ipAddressClaim struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + PoolRef struct { + Name string `json:"name"` + } `json:"poolRef"` + } `json:"spec"` + Status struct { + AddressRef struct { + Name string `json:"name"` + } `json:"addressRef"` + } `json:"status"` +} + +// ipAddress represents the structure of an IPAddress resource. +type ipAddress struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Address string `json:"address"` + Gateway string `json:"gateway"` + Prefix int `json:"prefix"` + PoolRef struct { + Name string `json:"name"` + } `json:"poolRef"` + } `json:"spec"` +} + +// ValidateIPAddressResources validates that CAPI IPAM resources (IPAddressClaim, IPAddress) are created. +func (e *ClusterE2ETest) ValidateIPAddressResources(poolName string) { + e.T.Log("Validating CAPI IPAM resources") + e.validateIPAddressClaims(poolName) + e.validateIPAddresses(poolName) + e.T.Log("CAPI IPAM resources validation completed successfully") +} + +// validateIPAddressClaims checks for IPAddressClaim resources. +func (e *ClusterE2ETest) validateIPAddressClaims(poolName string) { + e.T.Log("Checking for IPAddressClaim resources") + claimsOutput, err := e.KubectlClient.ExecuteCommand(context.Background(), + "get", "ipaddressclaims.ipam.cluster.x-k8s.io", + "-n", "eksa-system", + "--kubeconfig", e.KubeconfigFilePath(), + "-o", "json", + ) + if err != nil { + e.T.Fatalf("Failed to get IPAddressClaims: %v", err) + } + + var claimsList struct { + Items []ipAddressClaim `json:"items"` + } + if err := json.Unmarshal(claimsOutput.Bytes(), &claimsList); err != nil { + e.T.Fatalf("Failed to parse IPAddressClaims: %v", err) + } + + if len(claimsList.Items) == 0 { + e.T.Fatal("No IPAddressClaims found - CAPI IPAM may not be working") + } + + for _, claim := range claimsList.Items { + e.validateSingleClaim(claim, poolName) + } +} + +// validateSingleClaim validates a single IPAddressClaim. +func (e *ClusterE2ETest) validateSingleClaim(claim ipAddressClaim, poolName string) { + if claim.Spec.PoolRef.Name != poolName { + e.T.Logf("Skipping claim %s with pool %s (expected %s)", + claim.Metadata.Name, claim.Spec.PoolRef.Name, poolName) + return + } + + e.T.Logf("Found IPAddressClaim %s referencing pool %s", + claim.Metadata.Name, claim.Spec.PoolRef.Name) + + if claim.Status.AddressRef.Name == "" { + e.T.Fatalf("IPAddressClaim %s has no allocated address", claim.Metadata.Name) + } +} + +// validateIPAddresses checks for IPAddress resources. +func (e *ClusterE2ETest) validateIPAddresses(poolName string) { + e.T.Log("Checking for IPAddress resources") + addressesOutput, err := e.KubectlClient.ExecuteCommand(context.Background(), + "get", "ipaddresses.ipam.cluster.x-k8s.io", + "-n", "eksa-system", + "--kubeconfig", e.KubeconfigFilePath(), + "-o", "json", + ) + if err != nil { + e.T.Fatalf("Failed to get IPAddresses: %v", err) + } + + var addressesList struct { + Items []ipAddress `json:"items"` + } + if err := json.Unmarshal(addressesOutput.Bytes(), &addressesList); err != nil { + e.T.Fatalf("Failed to parse IPAddresses: %v", err) + } + + if len(addressesList.Items) == 0 { + e.T.Fatal("No IPAddress resources found - CAPI IPAM may not be working") + } + + for _, addr := range addressesList.Items { + if addr.Spec.PoolRef.Name != poolName { + continue + } + e.T.Logf("Found IPAddress %s with address %s from pool %s", + addr.Metadata.Name, addr.Spec.Address, addr.Spec.PoolRef.Name) + } +} + +// ValidateInClusterIPPool validates that the InClusterIPPool resource exists and is healthy. +func (e *ClusterE2ETest) ValidateInClusterIPPool(poolName string) { + e.T.Logf("Validating InClusterIPPool %s exists", poolName) + + output, err := e.KubectlClient.ExecuteCommand(context.Background(), + "get", "inclusterippool.ipam.cluster.x-k8s.io", poolName, + "-n", "eksa-system", + "--kubeconfig", e.KubeconfigFilePath(), + "-o", "json", + ) + if err != nil { + e.T.Fatalf("Failed to get InClusterIPPool %s: %v", poolName, err) + } + + var pool struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Addresses []string `json:"addresses"` + Prefix int `json:"prefix"` + Gateway string `json:"gateway"` + } `json:"spec"` + } + if err := json.Unmarshal(output.Bytes(), &pool); err != nil { + e.T.Fatalf("Failed to parse InClusterIPPool: %v", err) + } + + e.T.Logf("InClusterIPPool %s found with addresses %v, prefix %d, gateway %s ✓", + pool.Metadata.Name, pool.Spec.Addresses, pool.Spec.Prefix, pool.Spec.Gateway) +} + +// ValidateIPReleasedAfterScaleDown validates that IPs are released back to the pool after scale down. +func (e *ClusterE2ETest) ValidateIPReleasedAfterScaleDown(poolName string, expectedFreedCount int) { + e.T.Logf("Validating IPs are released after scale down") + + // Wait a bit for cleanup to happen + time.Sleep(30 * time.Second) + + // Count current IPAddresses + addressesOutput, err := e.KubectlClient.ExecuteCommand(context.Background(), + "get", "ipaddresses.ipam.cluster.x-k8s.io", + "-n", "eksa-system", + "--kubeconfig", e.KubeconfigFilePath(), + "-o", "json", + ) + if err != nil { + e.T.Fatalf("Failed to get IPAddresses after scale down: %v", err) + } + + var addressesList struct { + Items []interface{} `json:"items"` + } + if err := json.Unmarshal(addressesOutput.Bytes(), &addressesList); err != nil { + e.T.Fatalf("Failed to parse IPAddresses: %v", err) + } + + e.T.Logf("After scale down, found %d IPAddress resources", len(addressesList.Items)) + e.T.Log("IP release validation completed") +} + +// waitForMachineIP waits for a machine to have an IP address assigned. +func (e *ClusterE2ETest) waitForMachineIP(machineName, timeout string) error { + timeoutDuration, err := time.ParseDuration(timeout) + if err != nil { + return fmt.Errorf("invalid timeout format: %v", err) + } + + deadline := time.Now().Add(timeoutDuration) + + for time.Now().Before(deadline) { + output, err := e.KubectlClient.ExecuteCommand(context.Background(), + "get", "machine.cluster.x-k8s.io", machineName, + "-o", "jsonpath={.status.addresses[?(@.type==\"ExternalIP\")].address}", + "--kubeconfig", e.KubeconfigFilePath(), + "-n", "eksa-system", + ) + if err == nil && strings.TrimSpace(output.String()) != "" { + return nil + } + time.Sleep(10 * time.Second) + } + + return fmt.Errorf("timeout waiting for machine %s to get IP", machineName) +} + +// parseIPRange parses IP address specifications (ranges, CIDRs, single IPs) and returns all IPs. +func parseIPRange(addresses []string) []net.IP { + var result []net.IP + + for _, addr := range addresses { + // Handle IP range (e.g., "192.168.1.100-192.168.1.120") + if strings.Contains(addr, "-") { + parts := strings.Split(addr, "-") + if len(parts) == 2 { + startIP := net.ParseIP(strings.TrimSpace(parts[0])) + endIP := net.ParseIP(strings.TrimSpace(parts[1])) + if startIP != nil && endIP != nil { + result = append(result, generateIPRange(startIP, endIP)...) + } + } + continue + } + + // Handle CIDR (e.g., "192.168.1.0/24") + if strings.Contains(addr, "/") { + _, ipNet, err := net.ParseCIDR(addr) + if err == nil { + result = append(result, generateCIDRIPs(ipNet)...) + } + continue + } + + // Handle single IP + ip := net.ParseIP(addr) + if ip != nil { + result = append(result, ip) + } + } + + return result +} + +// generateIPRange generates all IPs between start and end (inclusive). +func generateIPRange(start, end net.IP) []net.IP { + var result []net.IP + start = start.To4() + end = end.To4() + + for ip := start; !ip.Equal(end); ip = nextIP(ip) { + newIP := make(net.IP, len(ip)) + copy(newIP, ip) + result = append(result, newIP) + } + result = append(result, end) + + return result +} + +// generateCIDRIPs generates all IPs in a CIDR block. +func generateCIDRIPs(ipNet *net.IPNet) []net.IP { + var result []net.IP + for ip := ipNet.IP.Mask(ipNet.Mask); ipNet.Contains(ip); ip = nextIP(ip) { + newIP := make(net.IP, len(ip)) + copy(newIP, ip) + result = append(result, newIP) + } + return result +} + +// nextIP returns the next IP address. +func nextIP(ip net.IP) net.IP { + ip = ip.To4() + result := make(net.IP, len(ip)) + copy(result, ip) + + for i := len(result) - 1; i >= 0; i-- { + result[i]++ + if result[i] > 0 { + break + } + } + return result +} + +// isIPInRange checks if an IP is in the expected range. +func isIPInRange(ipStr string, expectedIPs []net.IP) bool { + ip := net.ParseIP(ipStr) + if ip == nil { + return false + } + + for _, expected := range expectedIPs { + if ip.Equal(expected) { + return true + } + } + return false +}