feat: support mount SMB Azure File with user-provided OAuth token through k8s secret

pkg/azurefile/azurefile.go

@@ -173,8 +173,11 @@ const (
 	runtimeClassHandlerField          = "runtimeclasshandler"
 	defaultRuntimeClassHandler        = "kata-cc"
 	mountWithManagedIdentityField     = "mountwithmanagedidentity"
+	mountWithOAuthTokenField          = "mountwithoauthtoken"
 	mountWithWITokenField             = "mountwithworkloadidentitytoken"
 
+	defaultSecretOAuthToken = "oauthtoken"
+
 	accountNotProvisioned = "StorageAccountIsNotProvisioned"
 	// this is a workaround fix for 429 throttling issue, will update cloud provider for better fix later
 	tooManyRequests   = "TooManyRequests"
@@ -292,6 +295,8 @@ type Driver struct {
 	secretCacheMap azcache.Resource
 	// a map storing all volumes using data plane API <volumeID, value>
 	dataPlaneAPIVolMap sync.Map
+	// a map storing OAuth token SHA per server to avoid unnecessary credential cache refresh
+	oauthTokenSHAMap sync.Map
 	// a timed cache storing all storage accounts that are using data plane API temporarily
 	dataPlaneAPIAccountCache azcache.Resource
 	// a timed cache storing account search history (solve account list throttling issue)
@@ -824,7 +829,7 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
 
 	var protocol, accountKey, secretName, pvcNamespace string
 	// getAccountKeyFromSecret indicates whether get account key only from k8s secret
-	var getAccountKeyFromSecret, getLatestAccountKey, mountWithManagedIdentity, mountWithWIToken bool
+	var getAccountKeyFromSecret, getLatestAccountKey, mountWithManagedIdentity, mountWithOAuthToken, mountWithWIToken bool
 	var clientID, tenantID, tokenFilePath, serviceAccountToken string
 
 	for k, v := range reqContext {
@@ -861,6 +866,10 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
 			if mountWithManagedIdentity, err = strconv.ParseBool(v); err != nil {
 				return rgName, accountName, accountKey, fileShareName, diskName, subsID, tenantID, tokenFilePath, fmt.Errorf("invalid %s: %s in volume context", mountWithManagedIdentityField, v)
 			}
+		case mountWithOAuthTokenField:
+			if mountWithOAuthToken, err = strconv.ParseBool(v); err != nil {
+				return rgName, accountName, accountKey, fileShareName, diskName, subsID, tenantID, tokenFilePath, fmt.Errorf("invalid %s: %s in volume context", mountWithOAuthTokenField, v)
+			}
 		case mountWithWITokenField:
 			if mountWithWIToken, err = strconv.ParseBool(v); err != nil {
 				return rgName, accountName, accountKey, fileShareName, diskName, subsID, tenantID, tokenFilePath, fmt.Errorf("invalid %s: %s in volume context", mountWithWITokenField, v)
@@ -905,6 +914,21 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
 		return rgName, accountName, accountKey, fileShareName, diskName, subsID, tenantID, tokenFilePath, nil
 	}
 
+	if mountWithOAuthToken {
+		klog.V(2).Infof("mountWithOAuthToken is true, use OAuth token from secret for mount")
+		// Read accountName from secret if not already set
+		if accountName == "" && secretName != "" {
+			name, _, _, err := d.GetStorageAccountFromSecret(ctx, secretName, secretNamespace)
+			if err != nil {
+				return rgName, accountName, accountKey, fileShareName, diskName, subsID, tenantID, tokenFilePath, fmt.Errorf("failed to get account name from secret for mountWithOAuthToken: %v", err)
+			}
+			if name != "" {
+				accountName = name
+			}
+		}
+		return rgName, accountName, accountKey, fileShareName, diskName, subsID, tenantID, tokenFilePath, nil
+	}
+
 	if mountWithWIToken {
 		if clientID == "" {
 			clientID = d.cloud.Config.AzureAuthConfig.UserAssignedIdentityID
@@ -957,7 +981,7 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
 			if secretName != "" {
 				var name string
 				// 2. if not found in cache, get account key from kubernetes secret
-				name, accountKey, err = d.GetStorageAccountFromSecret(ctx, secretName, secretNamespace)
+				name, accountKey, _, err = d.GetStorageAccountFromSecret(ctx, secretName, secretNamespace)
 				if name != "" {
 					accountName = name
 				}
@@ -1354,7 +1378,7 @@ func (d *Driver) GetStorageAccesskey(ctx context.Context, accountOptions *storag
 	if secretName == "" {
 		secretName = fmt.Sprintf(secretNameTemplate, accountName)
 	}
-	_, accountKey, err := d.GetStorageAccountFromSecret(ctx, secretName, secretNamespace)
+	_, accountKey, _, err := d.GetStorageAccountFromSecret(ctx, secretName, secretNamespace)
 	if err != nil {
 		klog.V(2).Infof("could not get account(%s) key from secret(%s), error: %v, use cluster identity to get account key instead", accountOptions.Name, secretName, err)
 		accountKey, err = d.GetStorageAccesskeyWithSubsID(ctx, accountOptions.SubscriptionID, accountOptions.Name, accountOptions.ResourceGroup, accountOptions.GetLatestAccountKey)
@@ -1378,21 +1402,22 @@ func (d *Driver) GetStorageAccesskeyWithSubsID(ctx context.Context, subsID, acco
 	return d.cloud.GetStorageAccesskey(ctx, accountClient, account, resourceGroup, getLatestAccountKey)
 }
 
-// GetStorageAccountFromSecret get storage account key from k8s secret
-// return <accountName, accountKey, error>
-func (d *Driver) GetStorageAccountFromSecret(ctx context.Context, secretName, secretNamespace string) (string, string, error) {
+// GetStorageAccountFromSecret get storage account key and OAuth token from k8s secret
+// return <accountName, accountKey, oauthToken, error>
+func (d *Driver) GetStorageAccountFromSecret(ctx context.Context, secretName, secretNamespace string) (string, string, string, error) {
 	if d.kubeClient == nil {
-		return "", "", fmt.Errorf("could not get account key from secret(%s): KubeClient is nil", secretName)
+		return "", "", "", fmt.Errorf("could not get credentials from secret(%s): KubeClient is nil", secretName)
 	}
 
 	secret, err := d.kubeClient.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{})
 	if err != nil {
-		return "", "", fmt.Errorf("could not get secret(%v): %v", secretName, err)
+		return "", "", "", fmt.Errorf("could not get secret(%v): %v", secretName, err)
 	}
 
-	accountName := strings.TrimSpace(string(secret.Data[defaultSecretAccountName][:]))
-	accountKey := strings.TrimSpace(string(secret.Data[defaultSecretAccountKey][:]))
-	return accountName, accountKey, nil
+	accountName := strings.TrimSpace(string(secret.Data[defaultSecretAccountName]))
+	accountKey := strings.TrimSpace(string(secret.Data[defaultSecretAccountKey]))
+	oauthToken := strings.TrimSpace(string(secret.Data[defaultSecretOAuthToken]))
+	return accountName, accountKey, oauthToken, nil
 }
 
 // getSubnetResourceID get default subnet resource ID from cloud provider config

pkg/azurefile/controllerserver.go

@@ -126,7 +126,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 	}
 	var sku, subsID, resourceGroup, location, account, fileShareName, diskName, fsType, secretName string
 	var secretNamespace, pvcNamespace, protocol, customTags, storageEndpointSuffix, networkEndpointType, shareAccessTier, accountAccessTier, rootSquashType, tagValueDelimiter string
-	var createAccount, useSeretCache, matchTags, selectRandomMatchingAccount, getLatestAccountKey, encryptInTransit, mountWithManagedIdentity, mountWithWIToken bool
+	var createAccount, useSeretCache, matchTags, selectRandomMatchingAccount, getLatestAccountKey, encryptInTransit, mountWithManagedIdentity, mountWithWIToken, mountWithOAuthToken bool
 	var vnetResourceGroup, vnetName, vnetLinkName, publicNetworkAccess, subnetName, shareNamePrefix, fsGroupChangePolicy, useDataPlaneAPI, privateDNSZoneResourceGroup string
 	var requireInfraEncryption, disableDeleteRetentionPolicy, enableLFS, isMultichannelEnabled, allowSharedKeyAccess, allowCrossTenantReplication *bool
 	var provisionedBandwidthMibps, provisionedIops *int32
@@ -327,20 +327,33 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 			if err != nil {
 				return nil, status.Errorf(codes.InvalidArgument, "invalid %s: %s in storage class", mountWithWITokenField, v)
 			}
+		case mountWithOAuthTokenField:
+			mountWithOAuthToken, err = strconv.ParseBool(v)
+			if err != nil {
+				return nil, status.Errorf(codes.InvalidArgument, "invalid %s: %s in storage class", mountWithOAuthTokenField, v)
+			}
 		default:
 			return nil, status.Errorf(codes.InvalidArgument, "invalid parameter %q in storage class", k)
 		}
 	}
 
-	if mountWithManagedIdentity && mountWithWIToken {
-		return nil, status.Error(codes.InvalidArgument, "mountwithmanagedidentity and mountwithworkloadidentitytoken cannot be both true in storage class")
+	if (mountWithManagedIdentity && mountWithWIToken) || (mountWithManagedIdentity && mountWithOAuthToken) || (mountWithWIToken && mountWithOAuthToken) {
+		return nil, status.Errorf(codes.InvalidArgument, "only one of %s, %s, and %s can be true in storage class", mountWithManagedIdentityField, mountWithOAuthTokenField, mountWithWITokenField)
+	}
+
+	if mountWithOAuthToken && secretName == "" {
+		return nil, status.Errorf(codes.InvalidArgument, "%s is required when %s is true", secretNameField, mountWithOAuthTokenField)
+	}
+
+	if mountWithOAuthToken && (protocol == nfs || fsType == nfs) {
+		return nil, status.Errorf(codes.InvalidArgument, "%s is not supported with NFS protocol", mountWithOAuthTokenField)
 	}
 
-	// When using managed identity or workload identity token for mount,
-	// the account key should not be stored in the secret since mount
-	// authentication uses identity-based tokens, not account keys.
-	if mountWithManagedIdentity || mountWithWIToken {
+	var requiresSmbOAuth *bool
+	if mountWithManagedIdentity || mountWithWIToken || mountWithOAuthToken {
 		storeAccountKey = false
+		klog.V(2).Info("enabling smb oauth for identity-based mount")
+		requiresSmbOAuth = to.Ptr(true)
 	}
 
 	if matchTags && account != "" {
@@ -572,12 +585,6 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 		}
 	}
 
-	var requiresSmbOAuth *bool
-	if mountWithManagedIdentity || mountWithWIToken {
-		klog.V(2).Info("enabling smb oauth for managed identity or work identity token based mount")
-		requiresSmbOAuth = to.Ptr(true)
-	}
-
 	accountOptions := &storage.AccountOptions{
 		Name:                                    account,
 		Type:                                    sku,

pkg/azurefile/controllerserver_test.go

@@ -1374,7 +1374,7 @@ var _ = ginkgo.Describe("TestCreateVolume", func() {
 					},
 				}
 
-				expectedErr := status.Errorf(codes.InvalidArgument, "%s and %s cannot be both true in storage class", mountWithManagedIdentityField, mountWithWITokenField)
+				expectedErr := status.Errorf(codes.InvalidArgument, "only one of %s, %s, and %s can be true in storage class", mountWithManagedIdentityField, mountWithOAuthTokenField, mountWithWITokenField)
 				_, err := d.CreateVolume(ctx, req)
 				gomega.Expect(err).To(gomega.Equal(expectedErr))
 			})