diff --git a/internal/command/e2etest/pluggable_state_store_test.go b/internal/command/e2etest/pluggable_state_store_test.go index 208a8001fbd7..92e6e1953c1c 100644 --- a/internal/command/e2etest/pluggable_state_store_test.go +++ b/internal/command/e2etest/pluggable_state_store_test.go @@ -679,3 +679,141 @@ func TestPrimary_stateStore_providerCmds(t *testing.T) { // This test just asserts that `terraform providers schema` can read state // via the state store, and therefore detects all 3 providers needed for the output. } + +func TestInit_pluggableStateStore_providerUpgrade(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } + + t.Setenv(e2e.TestExperimentFlag, "true") + terraformBin := e2e.GoBuild("github.com/hashicorp/terraform", "terraform") + fixturePath := filepath.Join("testdata", "full-workflow-with-unconfigured-state-store-fs") + tf := e2e.NewBinary(t, terraformBin, fixturePath) + + // Add a state file describing a resource from the simple (v5) provider, so + // we can test that the state is read and used to get all the provider schemas + fakeState := states.BuildState(func(s *states.SyncState) { + s.SetResourceInstanceCurrent( + addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "simple_resource", + Name: "foo", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + &states.ResourceInstanceObjectSrc{ + AttrsJSON: []byte(`{"id":"bar"}`), + Status: states.ObjectReady, + }, + addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("simple6"), + Module: addrs.RootModule, + }, + ) + }) + fakeStateFile := &statefile.File{ + Lineage: "boop", + Serial: 4, + TerraformVersion: version.Must(version.NewVersion("1.0.0")), + State: fakeState, + } + var fakeStateBuf bytes.Buffer + err := statefile.WriteForTest(fakeStateFile, &fakeStateBuf) + if err != nil { + t.Error(err) + } + fakeStateBytes := fakeStateBuf.Bytes() + + oldStateDirName := "old" + newStateDirName := "new" + + if err := os.MkdirAll(filepath.Join(tf.WorkDir(), oldStateDirName, "terraform.tfstate.d", "default"), os.ModePerm); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tf.WorkDir(), oldStateDirName, "terraform.tfstate.d", "default", "terraform.tfstate"), fakeStateBytes, 0644); err != nil { + t.Fatal(err) + } + + platform := getproviders.CurrentPlatform.String() + cachePath := tf.Path("cache") + fsMirrorPath := filepath.Join(cachePath, "registry.terraform.io/hashicorp/simple6") + + // build old version + oldProviderTmp := filepath.Join(tf.WorkDir(), "terraform-provider-simple6") + oldPath := filepath.Join(fsMirrorPath, "1.2.3", platform) + oldProviderExe := e2e.GoBuild( + "github.com/hashicorp/terraform/internal/provider-simple-v6/main", oldProviderTmp, + "-ldflags", "-X 'main.parentStateDir="+oldStateDirName+"'") + if err := os.MkdirAll(oldPath, os.ModePerm); err != nil { + t.Fatal(err) + } + targetPath := filepath.Join(oldPath, "terraform-provider-simple6") + if err := os.Rename(oldProviderExe, targetPath); err != nil { + t.Fatal(err) + } + + // first init with old version + stdout, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir="+cachePath, "-no-color") + if err != nil { + t.Fatalf("unexpected error: %s\nstderr:\n%s", err, stderr) + } + + firstExpectedMsgs := []string{ + "Installed hashicorp/simple6 v1.2.3", + "Using previously-installed hashicorp/simple6 v1.2.3", + } + for _, msg := range firstExpectedMsgs { + if !strings.Contains(stdout, msg) { + t.Errorf("unexpected output from first init, expected %q, but got:\n%s", msg, stdout) + } + } + + // build new version + newProviderTmp := filepath.Join(tf.WorkDir(), "terraform-provider-simple6") + newPath := filepath.Join(fsMirrorPath, "1.5.0", platform) + newProviderExe := e2e.GoBuild( + "github.com/hashicorp/terraform/internal/provider-simple-v6/main", newProviderTmp, + "-ldflags", "-X 'main.parentStateDir="+newStateDirName+"'") + if err := os.MkdirAll(newPath, os.ModePerm); err != nil { + t.Fatal(err) + } + targetPath = filepath.Join(newPath, "terraform-provider-simple6") + if err := os.Rename(newProviderExe, targetPath); err != nil { + t.Fatal(err) + } + + backendStateFile := filepath.Join(tf.WorkDir(), ".terraform", "terraform.tfstate") + _, err = os.ReadFile(backendStateFile) + if err != nil { + t.Fatalf("failed to open backend state file: %s", err) + } + + // second init with new version in place + stdout, stderr, err = tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir="+cachePath, "-no-color", "-upgrade", "-force-copy") + if err != nil { + t.Fatalf("unexpected error: %s\nstderr:\n%s", err, stderr) + } + + secondExpectedMsgs := []string{ + "Installed hashicorp/simple6 v1.5.0", + "Using previously-installed hashicorp/simple6 v1.5.0", + } + for _, msg := range secondExpectedMsgs { + if !strings.Contains(stdout, msg) { + t.Errorf("unexpected output from second init, expected %q, but got:\n%s", msg, stdout) + } + } + + fi, err := os.Stat(path.Join(tf.WorkDir(), newStateDirName, "terraform.tfstate.d", "default", "terraform.tfstate")) + if err != nil { + t.Fatalf("failed to open new default workspace's state file: %s", err) + } + if fi.Size() == 0 { + t.Fatal("default workspace's state file should not have size 0 bytes") + } + + // TODO: check if both files match +} diff --git a/internal/command/e2etest/testdata/full-workflow-with-unconfigured-state-store-fs/main.tf b/internal/command/e2etest/testdata/full-workflow-with-unconfigured-state-store-fs/main.tf new file mode 100644 index 000000000000..d2f5c9b4446f --- /dev/null +++ b/internal/command/e2etest/testdata/full-workflow-with-unconfigured-state-store-fs/main.tf @@ -0,0 +1,23 @@ +terraform { + required_providers { + simple6 = { + source = "registry.terraform.io/hashicorp/simple6" + } + } + + state_store "simple6_fs" { + provider "simple6" {} + } +} + +variable "name" { + default = "world" +} + +resource "terraform_data" "my-data" { + input = "hello ${var.name}" +} + +output "greeting" { + value = resource.terraform_data.my-data.output +} diff --git a/internal/command/init.go b/internal/command/init.go index 7033efba00da..760ee5bfe37f 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -158,7 +158,7 @@ func (c *InitCommand) initCloud(ctx context.Context, root *configs.Module, extra return back, true, diags } -func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, initArgs *arguments.Init, configLocks *depsfile.Locks, view views.Init) (be backend.Backend, output bool, diags tfdiags.Diagnostics) { +func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, initArgs *arguments.Init, previousLocks *depsfile.Locks, configLocks *depsfile.Locks, view views.Init) (be backend.Backend, output bool, diags tfdiags.Diagnostics) { ctx, span := tracer.Start(ctx, "initialize backend") _ = ctx // prevent staticcheck from complaining to avoid a maintenance hazard of having the wrong ctx in scope here defer span.End() @@ -234,6 +234,7 @@ func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, ini opts = &BackendOpts{ StateStoreConfig: root.StateStore, Locks: configLocks, + PreviousLocks: previousLocks, ConfigOverride: configOverride, Init: true, ViewType: initArgs.ViewType, diff --git a/internal/command/init_run.go b/internal/command/init_run.go index d63013f94eda..1e4d82c09bcd 100644 --- a/internal/command/init_run.go +++ b/internal/command/init_run.go @@ -234,10 +234,10 @@ func (c *InitCommand) run(initArgs *arguments.Init, view views.Init) int { case initArgs.Cloud && rootModEarly.CloudConfig != nil: back, backendOutput, backDiags = c.initCloud(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, view) case initArgs.Backend: - back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs, configLocks, view) + back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs, previousLocks, configLocks, view) default: // load the previously-stored backend config - back, backDiags = c.Meta.backendFromState(ctx) + back, backDiags = c.Meta.backendFromState(ctx, previousLocks) } if backendOutput { header = true diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 710558b8001c..2768909fbdc4 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -65,6 +65,12 @@ type BackendOpts struct { // version in use currently. Locks *depsfile.Locks + // PreviousLocks allows state-migration logic to use the older version of the provider + // during upgrade-triggered migrations. + PreviousLocks *depsfile.Locks + + SkipGetProviderSchemaCache bool + // ConfigOverride is an hcl.Body that, if non-nil, will be used with // configs.MergeBodies to override the type-specific backend configuration // arguments in Config. @@ -370,7 +376,7 @@ func (m *Meta) BackendForLocalPlan(plan *plans.Plan) (backendrun.OperationsBacke return nil, diags } - factories, err := m.ProviderFactoriesFromLocks(locks) + factories, err := m.ProviderFactoriesFromLocks(locks, false) if err != nil { // This may happen if the provider isn't present in the provider cache. // This should be caught earlier by logic that diffs the config against the backend state file. @@ -1044,7 +1050,7 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di v := views.NewInit(opts.ViewType, m.View) v.Output(views.InitMessageCode("state_store_unset"), s.StateStore.Type) - return m.stateStore_to_backend(sMgr, "local", localB, nil, opts.ViewType) + return m.stateStore_to_backend(sMgr, "local", localB, nil, opts) // Configuring a backend for the first time or -reconfigure flag was used case backendConfig != nil && s.Backend.Empty() && @@ -1118,7 +1124,7 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di newBackendCfgState.SetConfig(configVal, b.ConfigSchema()) newBackendCfgState.Hash = uint64(cHash) - return m.stateStore_to_backend(sMgr, backendConfig.Type, b, newBackendCfgState, opts.ViewType) + return m.stateStore_to_backend(sMgr, backendConfig.Type, b, newBackendCfgState, opts) // Migration from backend to state store case backendConfig == nil && !s.Backend.Empty() && @@ -1220,7 +1226,7 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di // AND we're not providing any overrides. An override can mean a change overriding an unchanged backend block (indicated by the hash value). if (uint64(cHash) == s.StateStore.Hash) && (!opts.Init || opts.ConfigOverride == nil) { log.Printf("[TRACE] Meta.Backend: using already-initialized, unchanged %q state_store configuration", stateStoreConfig.Type) - savedStateStore, sssDiags := m.savedStateStore(sMgr) + savedStateStore, sssDiags := m.savedStateStore(sMgr, opts.Locks, false) diags = diags.Append(sssDiags) // Verify that selected workspace exist. Otherwise prompt user to create one if opts.Init && savedStateStore != nil { @@ -1238,7 +1244,7 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di // don't need to migrate, we update the state store cache hash value. if !m.stateStoreConfigNeedsMigration(stateStoreConfig, s.StateStore, opts) { log.Printf("[TRACE] Meta.Backend: using already-initialized %q state store configuration", stateStoreConfig.Type) - savedStateStore, moreDiags := m.savedStateStore(sMgr) + savedStateStore, moreDiags := m.savedStateStore(sMgr, opts.Locks, false) diags = diags.Append(moreDiags) if moreDiags.HasErrors() { return nil, diags @@ -1305,6 +1311,20 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di return nil, diags } + pLock := opts.Locks.Provider(stateStoreConfig.ProviderAddr) + lockVersion, err := providerreqs.GoVersionFromVersion(pLock.Version()) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Warning, + "Unable to determine version of the state store provider", + fmt.Sprintf("Failed to parse version of %s from the lock file: %s", stateStoreConfig.ProviderAddr.ForDisplay(), err), + )) + return nil, diags + } + if !lockVersion.Equal(s.StateStore.Provider.Version) { + opts.SkipGetProviderSchemaCache = true + } + return m.stateStore_changed(stateStoreConfig, cHash, sMgr, opts, initReason) default: @@ -1467,7 +1487,7 @@ func (m *Meta) determineStateStoreInitReason(cfgState *workdir.StateStoreConfigS // from the backend state. This should be used only when a user runs // `terraform init -backend=false`. This function returns a local backend if // there is no backend state or no backend configured. -func (m *Meta) backendFromState(_ context.Context) (backend.Backend, tfdiags.Diagnostics) { +func (m *Meta) backendFromState(_ context.Context, locks *depsfile.Locks) (backend.Backend, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics // Get the path to where we store a local cache of backend configuration // if we're using a remote backend. This may not yet exist which means @@ -1493,7 +1513,7 @@ func (m *Meta) backendFromState(_ context.Context) (backend.Backend, tfdiags.Dia // state_store log.Printf("[TRACE] Meta.Backend: working directory was previously initialized for %q state store", s.StateStore.Type) var ssDiags tfdiags.Diagnostics - b, ssDiags = m.savedStateStore(sMgr) // Relies on the state manager's internal state being refreshed above. + b, ssDiags = m.savedStateStore(sMgr, locks, false) // Relies on the state manager's internal state being refreshed above. diags = diags.Append(ssDiags) if ssDiags.HasErrors() { return nil, diags @@ -2531,17 +2551,17 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, stateStoreHash int, backend } // Migrating a state store to backend (including local). -func (m *Meta) stateStore_to_backend(ssSMgr *clistate.LocalState, dstBackendType string, dstBackend backend.Backend, newBackendState *workdir.BackendConfigState, viewType arguments.ViewType) (backend.Backend, tfdiags.Diagnostics) { +func (m *Meta) stateStore_to_backend(ssSMgr *clistate.LocalState, dstBackendType string, dstBackend backend.Backend, newBackendState *workdir.BackendConfigState, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics s := ssSMgr.State() stateStoreType := s.StateStore.Type - view := views.NewInit(viewType, m.View) + view := views.NewInit(opts.ViewType, m.View) view.Output(views.StateMigrateLocalMessage, stateStoreType) // Initialize the configured state store - ss, moreDiags := m.savedStateStore(ssSMgr) + ss, moreDiags := m.savedStateStore(ssSMgr, opts.Locks, false) diags = diags.Append(moreDiags) if moreDiags.HasErrors() { return nil, diags @@ -2553,7 +2573,7 @@ func (m *Meta) stateStore_to_backend(ssSMgr *clistate.LocalState, dstBackendType DestinationType: dstBackendType, Source: ss, Destination: dstBackend, - ViewType: viewType, + ViewType: opts.ViewType, }) if err != nil { diags = diags.Append(err) @@ -2668,7 +2688,7 @@ func (m *Meta) stateStore_changed(cfg *configs.StateStore, cfgHash int, sMgr *cl } // Grab the source state store - srcB, srcBDiags := m.savedStateStore(sMgr) + srcB, srcBDiags := m.savedStateStore(sMgr, opts.PreviousLocks, opts.SkipGetProviderSchemaCache) diags = diags.Append(srcBDiags) if srcBDiags.HasErrors() { return nil, diags @@ -2685,11 +2705,12 @@ func (m *Meta) stateStore_changed(cfg *configs.StateStore, cfgHash int, sMgr *cl // Perform the migration err := m.backendMigrateState(&backendMigrateOpts{ - SourceType: s.StateStore.Type, - DestinationType: cfg.Type, - Source: srcB, - Destination: dstB, - ViewType: vt, + SkipGetProviderSchemaCache: opts.SkipGetProviderSchemaCache, + SourceType: s.StateStore.Type, + DestinationType: cfg.Type, + Source: srcB, + Destination: dstB, + ViewType: vt, }) if err != nil { diags = diags.Append(err) @@ -2822,7 +2843,7 @@ To make the initial dependency selections that will initialize the dependency lo } // Initializing a saved state store from the backend state file (aka 'cache file', aka 'legacy state file') -func (m *Meta) savedStateStore(sMgr *clistate.LocalState) (backend.Backend, tfdiags.Diagnostics) { +func (m *Meta) savedStateStore(sMgr *clistate.LocalState, locks *depsfile.Locks, skipCache bool) (backend.Backend, tfdiags.Diagnostics) { // We're preparing a state_store version of backend.Backend. // // The provider and state store will be configured using the backend state file. @@ -2831,7 +2852,7 @@ func (m *Meta) savedStateStore(sMgr *clistate.LocalState) (backend.Backend, tfdi s := sMgr.State() - factory, pDiags := m.StateStoreProviderFactoryFromConfigState(s.StateStore) + factory, pDiags := m.StateStoreProviderFactoryFromConfigState(s.StateStore, locks, skipCache) diags = diags.Append(pDiags) if pDiags.HasErrors() { return nil, diags @@ -3313,7 +3334,7 @@ func (m *Meta) StateStoreProviderFactoryFromConfig(config *configs.StateStore, l }) } - factories, err := m.ProviderFactoriesFromLocks(locks) + factories, err := m.ProviderFactoriesFromLocks(locks, false) if err != nil { // This may happen if the provider isn't present in the provider cache. // This should be caught earlier by logic that diffs the config against the backend state file. @@ -3346,7 +3367,7 @@ func (m *Meta) StateStoreProviderFactoryFromConfig(config *configs.StateStore, l return factory, diags } -func (m *Meta) StateStoreProviderFactoryFromConfigState(cfgState *workdir.StateStoreConfigState) (providers.Factory, tfdiags.Diagnostics) { +func (m *Meta) StateStoreProviderFactoryFromConfigState(cfgState *workdir.StateStoreConfigState, locks *depsfile.Locks, skipCache bool) (providers.Factory, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics if cfgState == nil { @@ -3362,7 +3383,7 @@ func (m *Meta) StateStoreProviderFactoryFromConfigState(cfgState *workdir.StateS }) } - factories, err := m.ProviderFactories() + factories, err := m.providerFactoriesFromLocks(locks, skipCache) if err != nil { // This may happen if the provider isn't present in the provider cache. // This should be caught earlier by logic that diffs the config against the backend state file. diff --git a/internal/command/meta_backend_migrate.go b/internal/command/meta_backend_migrate.go index 38bb09f91d15..b3090928fe96 100644 --- a/internal/command/meta_backend_migrate.go +++ b/internal/command/meta_backend_migrate.go @@ -28,6 +28,7 @@ import ( type backendMigrateOpts struct { SourceType, DestinationType string + SkipGetProviderSchemaCache bool Source, Destination backend.Backend ViewType arguments.ViewType @@ -496,7 +497,8 @@ func (m *Meta) backendMigrateEmptyConfirm(source, destination statemgr.Full, opt } func (m *Meta) backendMigrateNonEmptyConfirm( - sourceState, destinationState statemgr.Full, opts *backendMigrateOpts) (bool, error) { + sourceState, destinationState statemgr.Full, opts *backendMigrateOpts, +) (bool, error) { // We need to grab both states so we can write them to a file source := sourceState.State() destination := destinationState.State() @@ -580,7 +582,7 @@ func (m *Meta) backendMigrateTFC(opts *backendMigrateOpts) error { if err != nil { return err } - //to be used below, not yet implamented + // to be used below, not yet implamented // destinationWorkspaces, destinationSingleState _, _, err = retrieveWorkspaces(opts.Destination, opts.SourceType) if err != nil { @@ -639,7 +641,7 @@ func (m *Meta) backendMigrateTFC(opts *backendMigrateOpts) error { if migrate, err := m.promptSingleToCloudSingleStateMigration(opts); err != nil { return err } else if !migrate { - return nil //skip migrating but return successfully + return nil // skip migrating but return successfully } return m.backendMigrateState_s_s(opts) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 043f81cc93db..5c9c197f32c6 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -2159,7 +2159,12 @@ func TestBackendFromState(t *testing.T) { // them to match just for this test. wd.OverrideDataDir(".") - stateBackend, diags := m.backendFromState(context.Background()) + locks, diags := m.lockedDependencies() + if diags.HasErrors() { + t.Fatal(diags.Err()) + } + + stateBackend, diags := m.backendFromState(context.Background(), locks) if diags.HasErrors() { t.Fatal(diags.Err()) } @@ -2419,8 +2424,13 @@ func TestSavedStateStore(t *testing.T) { t.Fatalf("unexpected error: %s", err) } + locks, diags := m.lockedDependencies() + if diags.HasErrors() { + t.Fatal(diags.Err()) + } + // Code under test - b, diags := m.savedStateStore(sMgr) + b, diags := m.savedStateStore(sMgr, locks, false) if diags.HasErrors() { t.Fatalf("unexpected errors: %s", diags.Err()) } @@ -2459,7 +2469,12 @@ func TestSavedStateStore(t *testing.T) { t.Fatalf("unexpected error: %s", err) } - _, diags := m.savedStateStore(sMgr) + locks, dDiags := m.lockedDependencies() + if dDiags.HasErrors() { + t.Fatal(dDiags.Err()) + } + + _, diags := m.savedStateStore(sMgr, locks, false) if !diags.HasErrors() { t.Fatal("expected errors but got none") } @@ -2496,7 +2511,12 @@ func TestSavedStateStore(t *testing.T) { t.Fatalf("unexpected error: %s", err) } - _, diags := m.savedStateStore(sMgr) + locks, dDiags := m.lockedDependencies() + if dDiags.HasErrors() { + t.Fatal(dDiags.Err()) + } + + _, diags := m.savedStateStore(sMgr, locks, false) if !diags.HasErrors() { t.Fatal("expected errors but got none") } diff --git a/internal/command/meta_providers.go b/internal/command/meta_providers.go index fde640485c01..c1ba5c4f78ed 100644 --- a/internal/command/meta_providers.go +++ b/internal/command/meta_providers.go @@ -280,7 +280,9 @@ func (m *Meta) ProviderFactories() (map[addrs.Provider]providers.Factory, error) return nil, fmt.Errorf("failed to read dependency lock file: %s", diags.Err()) } - return m.providerFactoriesFromLocks(locks) + // Pass false as skipCache because this function is not used to launch 'old' versions of a provider + // during a state migration using state_store + return m.providerFactoriesFromLocks(locks, false) } // ProviderFactoriesFromLocks receives in memory locks and uses them to produce a map @@ -290,12 +292,12 @@ func (m *Meta) ProviderFactories() (map[addrs.Provider]providers.Factory, error) // ProviderFactoriesFromLocks should only be used if the calling code relies on locks // that have not yet been persisted to a dependency lock file on disk. Realistically, this // means only code in the init command should use this method. -func (m *Meta) ProviderFactoriesFromLocks(configLocks *depsfile.Locks) (map[addrs.Provider]providers.Factory, error) { +func (m *Meta) ProviderFactoriesFromLocks(configLocks *depsfile.Locks, skipCache bool) (map[addrs.Provider]providers.Factory, error) { // Ensure overrides and unmanaged providers are reflected in the returned list of factories, // while avoiding mutating the in-memory locks := m.annotateDependencyLocksWithOverrides(configLocks.DeepCopy()) - return m.providerFactoriesFromLocks(locks) + return m.providerFactoriesFromLocks(locks, skipCache) } // providerFactoriesFromLocks returns a map of provider factories from a given set of locks. @@ -304,7 +306,7 @@ func (m *Meta) ProviderFactoriesFromLocks(configLocks *depsfile.Locks) (map[addr // Instead, use: // * `ProviderFactoriesFromLocks` - for use when locks aren't yet persisted to a dependency lock file. // * `ProviderFactories` - for use when Terraform is guaranteed to read all necessary locks from a dependency lock file. -func (m *Meta) providerFactoriesFromLocks(locks *depsfile.Locks) (map[addrs.Provider]providers.Factory, error) { +func (m *Meta) providerFactoriesFromLocks(locks *depsfile.Locks, skipCache bool) (map[addrs.Provider]providers.Factory, error) { // We'll always run through all of our providers, even if one of them // encounters an error, so that we can potentially report multiple errors // where appropriate and so that callers can potentially make use of the @@ -390,7 +392,7 @@ func (m *Meta) providerFactoriesFromLocks(locks *depsfile.Locks) (map[addrs.Prov continue } } - factories[provider] = providerFactory(cached) + factories[provider] = providerFactory(cached, skipCache) } for provider, localDir := range devOverrideProviders { factories[provider] = devOverrideProviderFactory(provider, localDir) @@ -419,7 +421,7 @@ func (m *Meta) internalProviders() map[string]providers.Factory { // providerFactory produces a provider factory that runs up the executable // file in the given cache package and uses go-plugin to implement // providers.Interface against it. -func providerFactory(meta *providercache.CachedProvider) providers.Factory { +func providerFactory(meta *providercache.CachedProvider, skipCache bool) providers.Factory { return func() (providers.Interface, error) { execFile, err := meta.ExecutableFile() if err != nil { @@ -451,23 +453,25 @@ func providerFactory(meta *providercache.CachedProvider) providers.Factory { // store the client so that the plugin can kill the child process protoVer := client.NegotiatedVersion() - return finalizeFactoryPlugin(raw, protoVer, meta.Provider, client), nil + return finalizeFactoryPlugin(raw, protoVer, meta.Provider, skipCache, client), nil } } // finalizeFactoryPlugin completes the setup of a plugin dispensed by the rpc // client to be returned by the plugin factory. -func finalizeFactoryPlugin(rawPlugin any, protoVersion int, addr addrs.Provider, client *plugin.Client) providers.Interface { +func finalizeFactoryPlugin(rawPlugin any, protoVersion int, addr addrs.Provider, skipCache bool, client *plugin.Client) providers.Interface { switch protoVersion { case 5: p := rawPlugin.(*tfplugin.GRPCProvider) p.PluginClient = client p.Addr = addr + p.SkipCache = skipCache return p case 6: p := rawPlugin.(*tfplugin6.GRPCProvider) p.PluginClient = client p.Addr = addr + p.SkipCache = skipCache return p default: panic("unsupported protocol version") @@ -480,11 +484,12 @@ func devOverrideProviderFactory(provider addrs.Provider, localDir getproviders.P // doesn't actually care about the version, so we can leave it // unspecified: overridden providers are not explicitly versioned. log.Printf("[DEBUG] Provider %s is overridden to load from %s", provider, localDir) + skipCache := false return providerFactory(&providercache.CachedProvider{ Provider: provider, Version: getproviders.UnspecifiedVersion, PackageDir: string(localDir), - }) + }, skipCache) } // unmanagedProviderFactory produces a provider factory that uses the passed @@ -532,6 +537,7 @@ func unmanagedProviderFactory(provider addrs.Provider, reattach *plugin.Reattach // store the client so that the plugin can kill the child process protoVer := client.NegotiatedVersion() + skipCache := false switch protoVer { case 0, 5: // As of the 0.15 release, sdk.v2 doesn't include the protocol @@ -539,9 +545,9 @@ func unmanagedProviderFactory(provider addrs.Provider, reattach *plugin.Reattach // go-plugin), so client.NegotiatedVersion() always returns 0. We // assume that an unmanaged provider reporting protocol version 0 is // actually using proto v5 for backwards compatibility. - return finalizeFactoryPlugin(raw, 5, provider, client), nil + return finalizeFactoryPlugin(raw, 5, provider, skipCache, client), nil case 6: - return finalizeFactoryPlugin(raw, 6, provider, client), nil + return finalizeFactoryPlugin(raw, 6, provider, skipCache, client), nil default: return nil, fmt.Errorf("unsupported protocol version %d", protoVer) } diff --git a/internal/e2e/e2e.go b/internal/e2e/e2e.go index 2695d77d777b..e122d0982c56 100644 --- a/internal/e2e/e2e.go +++ b/internal/e2e/e2e.go @@ -244,7 +244,7 @@ func (b *binary) SetLocalState(state *states.State) error { return statefile.Write(sf, f) } -func GoBuild(pkgPath, tmpPrefix string) string { +func GoBuild(pkgPath, tmpPrefix string, buildArgs ...string) string { dir, prefix := filepath.Split(tmpPrefix) tmpFile, err := os.CreateTemp(dir, prefix) if err != nil { @@ -259,6 +259,7 @@ func GoBuild(pkgPath, tmpPrefix string) string { if exp := os.Getenv(TestExperimentFlag); exp != "" && exp != "false" { args = append(args, "-ldflags", "-X 'main.experimentsAllowed=yes'") } + args = append(args, buildArgs...) args = append(args, pkgPath) cmd := exec.Command("go", args...) cmd.Stderr = os.Stderr diff --git a/internal/plugin/grpc_provider.go b/internal/plugin/grpc_provider.go index 0d7a5902bc93..09fa8d68e599 100644 --- a/internal/plugin/grpc_provider.go +++ b/internal/plugin/grpc_provider.go @@ -64,6 +64,9 @@ type GRPCProvider struct { // but it may not always be available for alternative execute modes. Addr addrs.Provider + // SkipCache WIP + SkipCache bool + // Proto client use to make the grpc service calls. client proto.ProviderClient @@ -83,7 +86,7 @@ func (p *GRPCProvider) GetProviderSchema() providers.GetProviderSchemaResponse { defer p.mu.Unlock() // check the global cache if we can - if !p.Addr.IsZero() { + if !p.Addr.IsZero() && !p.SkipCache { if resp, ok := providers.SchemaCache.Get(p.Addr); ok && resp.ServerCapabilities.GetProviderSchemaOptional { logger.Trace("GRPCProvider: returning cached schema", p.Addr.String()) return resp diff --git a/internal/plugin6/grpc_provider.go b/internal/plugin6/grpc_provider.go index 9da603a24723..f1ec2e930bb8 100644 --- a/internal/plugin6/grpc_provider.go +++ b/internal/plugin6/grpc_provider.go @@ -71,6 +71,9 @@ type GRPCProvider struct { // but it may not always be available for alternative execute modes. Addr addrs.Provider + // SkipCache WIP + SkipCache bool + // Proto client use to make the grpc service calls. client proto6.ProviderClient @@ -93,7 +96,7 @@ func (p *GRPCProvider) GetProviderSchema() providers.GetProviderSchemaResponse { defer p.mu.Unlock() // check the global cache if we can - if !p.Addr.IsZero() { + if !p.Addr.IsZero() && !p.SkipCache { if resp, ok := providers.SchemaCache.Get(p.Addr); ok && resp.ServerCapabilities.GetProviderSchemaOptional { logger.Trace("GRPCProvider.v6: returning cached schema", p.Addr.String()) return resp @@ -208,7 +211,7 @@ func (p *GRPCProvider) GetProviderSchema() providers.GetProviderSchemaResponse { } // set the global cache if we can - if !p.Addr.IsZero() { + if !p.Addr.IsZero() && !p.SkipCache { providers.SchemaCache.Set(p.Addr, resp) } diff --git a/internal/provider-simple-v6/main/main.go b/internal/provider-simple-v6/main/main.go index 1007de52b4e8..ea3185a9bf89 100644 --- a/internal/provider-simple-v6/main/main.go +++ b/internal/provider-simple-v6/main/main.go @@ -7,13 +7,23 @@ import ( "github.com/hashicorp/terraform/internal/grpcwrap" plugin "github.com/hashicorp/terraform/internal/plugin6" simple "github.com/hashicorp/terraform/internal/provider-simple-v6" + "github.com/hashicorp/terraform/internal/providers" "github.com/hashicorp/terraform/internal/tfplugin6" ) +// parentStateDir can be overriden via ldflags +var parentStateDir = "" + func main() { + var p providers.Interface + if parentStateDir != "" { + p = simple.ProviderWithParentStatePath(parentStateDir) + } else { + p = simple.Provider() + } plugin.Serve(&plugin.ServeOpts{ GRPCProviderFunc: func() tfplugin6.ProviderServer { - return grpcwrap.Provider6(simple.Provider()) + return grpcwrap.Provider6(p) }, }) } diff --git a/internal/provider-simple-v6/provider.go b/internal/provider-simple-v6/provider.go index 0225868e5d16..11b2c82f88ff 100644 --- a/internal/provider-simple-v6/provider.go +++ b/internal/provider-simple-v6/provider.go @@ -56,6 +56,14 @@ func ProviderWithDefaultState() providers.Interface { return p } +// ProviderWithParentStatePath returns an instance of providers.Interface, +// where the path to the states directory is prepended with the given path. +func ProviderWithParentStatePath(path string) providers.Interface { + parentDir = path + p := provider() + return p +} + // provider returns an instance of simple func provider() simple { simpleResource := providers.Schema{ diff --git a/internal/provider-simple-v6/state_store_fs.go b/internal/provider-simple-v6/state_store_fs.go index 30ad2608b7aa..deaa0c056e08 100644 --- a/internal/provider-simple-v6/state_store_fs.go +++ b/internal/provider-simple-v6/state_store_fs.go @@ -25,6 +25,8 @@ import ( const fsStoreName = "simple6_fs" const defaultStatesDir = "terraform.tfstate.d" +var parentDir = "" + // FsStore allows storing state in the local filesystem. // // This state storage implementation differs from the old "local" backend in core, @@ -79,6 +81,10 @@ func (f *FsStore) ConfigureStateStore(req providers.ConfigureStateStoreRequest) f.statesDir = defaultStatesDir } + if parentDir != "" { + f.statesDir = filepath.Join(parentDir, f.statesDir) + } + if f.states == nil { f.states = make(map[string]*statemgr.Filesystem) } @@ -172,6 +178,7 @@ func (f *FsStore) ReadStateBytes(req providers.ReadStateBytesRequest) providers. // E.g. terraform.tfstate.d/foobar/terraform.tfstate path := f.getStatePath(req.StateId) + log.Printf("[DEBUG] ReadStateBytes: reading data from path %q", path) file, err := os.Open(path) fileExists := true @@ -225,6 +232,8 @@ func (f *FsStore) WriteStateBytes(req providers.WriteStateBytesRequest) provider // E.g. terraform.tfstate.d/foobar/terraform.tfstate path := f.getStatePath(req.StateId) + log.Printf("[DEBUG] WriteStateBytes: writing data to path %q", path) + // Create or open state file dir := f.getStateDir(req.StateId) err := os.MkdirAll(dir, 0755) diff --git a/internal/rpcapi/dependencies.go b/internal/rpcapi/dependencies.go index de729aac0b9c..d37e566eada1 100644 --- a/internal/rpcapi/dependencies.go +++ b/internal/rpcapi/dependencies.go @@ -717,11 +717,13 @@ func providerFactoriesForLocks(locks *depsfile.Locks, pluginsDir *providercache. p := raw.(*tfplugin.GRPCProvider) p.PluginClient = client p.Addr = addr + p.SkipCache = false return p, nil case 6: p := raw.(*tfplugin6.GRPCProvider) p.PluginClient = client p.Addr = addr + p.SkipCache = false return p, nil default: panic("unsupported protocol version")