Skip to content

feat: Supporting streaming components to the catalog #5914

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yhakbar merged 4 commits into from
Apr 17, 2026
Merged

feat: Supporting streaming components to the catalog #5914

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8ea87f0
feat: Adding streamed components instead of waiting for everything to…
yhakbar Apr 15, 2026
32a0f12
fix: Removing test env var
yhakbar Apr 15, 2026
07ac834
chore: Isolating redesign code
yhakbar Apr 16, 2026
552aa8b
fix: Addressing review feedback
yhakbar Apr 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 107 additions & 36 deletions internal/cli/commands/catalog/catalog_redesign.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@
import (
"context"
"fmt"
"runtime"

"github.com/gruntwork-io/terragrunt/internal/cli/commands/catalog/tui"
"golang.org/x/sync/errgroup"

"github.com/gruntwork-io/terragrunt/internal/cli/commands/catalog/tui/redesign"
"github.com/gruntwork-io/terragrunt/internal/configbridge"
"github.com/gruntwork-io/terragrunt/internal/services/catalog"
"github.com/gruntwork-io/terragrunt/internal/util"
"github.com/gruntwork-io/terragrunt/internal/services/catalog/module"
"github.com/gruntwork-io/terragrunt/pkg/config"
"github.com/gruntwork-io/terragrunt/pkg/log"
"github.com/gruntwork-io/terragrunt/pkg/options"
Expand All @@ -19,55 +22,123 @@
// It launches the TUI immediately with a loading screen, then runs source
// discovery and module loading in the background. When loading completes,
// the TUI transitions to the module list or shows a welcome screen.
func runRedesign(ctx context.Context, l log.Logger, opts *options.TerragruntOptions, repoURL string) error {

Check failure on line 25 in internal/cli/commands/catalog/catalog_redesign.go

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this method to reduce its Cognitive Complexity from 16 to the 15 allowed. 


See more on https://sonarcloud.io/project/issues?id=gruntwork-io_terragrunt&issues=AZ2TXdIlQMUhC3rffAcz&open=AZ2TXdIlQMUhC3rffAcz&pullRequest=5914
// If an explicit URL was passed via CLI, use the default path
if repoURL != "" {
return runDefault(ctx, l, opts, repoURL)
}

return tui.RunRedesign(ctx, l, opts, func(ctx context.Context, status tui.StatusFunc) (catalog.CatalogService, error) {
status("Scanning terragrunt.hcl files for module sources...")
return redesign.RunRedesign(
ctx, l, opts,
func(
ctx context.Context, status redesign.StatusFunc, moduleCh chan<- *module.Module,
) (catalog.CatalogService, error) {
svc := catalog.NewCatalogService(opts)

onModule := func(mod *module.Module) {
select {
case moduleCh <- mod:
case <-ctx.Done():
}
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

urlCh := make(chan string, 10) //nolint:mnd

g, gctx := errgroup.WithContext(ctx)

g.Go(func() error {
return discoverCatalogConfigURLs(gctx, l, opts, urlCh)
})

g.Go(func() error {
return discoverSourceFileURLs(gctx, l, opts, urlCh)
})

go func() {
_ = g.Wait()

// Create parsing context for source discovery and catalog config
ctx, pctx := configbridge.NewParsingContext(ctx, l, opts)
close(urlCh)
}()

// Discover source URLs from terraform.source in terragrunt.hcl files
discoveredURLs, err := DiscoverSourceURLs(ctx, l, pctx)
if err != nil {
l.Warnf("Failed to discover source URLs: %v", err)
}
status("Discovering catalog sources...")

// Also read catalog config if it exists
catalogCfg, catalogErr := config.ReadCatalogConfig(ctx, l, pctx)
if catalogErr != nil {
l.Debugf("No catalog config found: %v", catalogErr)
}
maxWorkers := max(1, min(opts.Parallelism, runtime.GOMAXPROCS(0)))

// Merge: catalog URLs first, then discovered URLs
var allURLs []string
if catalogCfg != nil {
allURLs = append(allURLs, catalogCfg.URLs...)
}
loaders, loadCtx := errgroup.WithContext(gctx)
loaders.SetLimit(maxWorkers)

allURLs = append(allURLs, discoveredURLs...)
allURLs = util.RemoveDuplicates(allURLs)
seen := make(map[string]struct{})

if len(allURLs) == 0 {
return nil, nil
}
for repoURL := range urlCh {
if _, ok := seen[repoURL]; ok {
continue
}

status(fmt.Sprintf("Found %d source(s), cloning repositories...", len(allURLs)))
seen[repoURL] = struct{}{}

// Load modules from all discovered repos
svc := catalog.NewCatalogService(opts)
svc.WithRepoURLs(allURLs)
loaders.Go(func() error {
if err := svc.LoadStreamingURL(loadCtx, l, repoURL, onModule); err != nil {
// Individual repo failures are non-critical — warn and
// continue so remaining repos can still load.
l.Warnf("Error loading %s: %v", repoURL, err)
}

return nil
})
}

if err := loaders.Wait(); err != nil {
return nil, fmt.Errorf("loading modules: %w", err)
}

if err := g.Wait(); err != nil {
return nil, fmt.Errorf("discovering sources: %w", err)
}

if len(svc.Modules()) == 0 {
return nil, nil
}

return svc, nil
Comment on lines +79 to +102
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot Apr 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Don't convert loader failures into "no sources found".

Every LoadStreamingURL failure is downgraded to a warning, and the callback returns nil, nil when no modules were loaded. If all repos fail to clone/load, the welcome flow will present an empty-catalog state instead of surfacing the actual failure.

🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@internal/cli/commands/catalog/catalog_redesign.go` around lines 75 - 96, The
code currently swallows errors from LoadStreamingURL (inside loaders.Go) and
converts all failures into warnings and ultimately a nil, nil return when
svc.Modules() is empty; change the anonymous function passed to loaders.Go to
return the error from svc.LoadStreamingURL instead of only logging it so
loaders.Wait() can surface failures, and then propagate errors from
loaders.Wait() and g.Wait() by returning them (rather than only l.Warnf) so
callers of this function receive real errors instead of seeing an empty catalog;
ensure references: the anonymous func using svc.LoadStreamingURL(loadCtx, l,
repoURL, onModule), loaders.Wait(), g.Wait(), and the svc.Modules() empty check
are updated accordingly.

})
}

if err := svc.Load(ctx, l); err != nil {
return svc, err
}
// discoverCatalogConfigURLs reads catalog URLs from the root config and
// sends each to urlCh.
func discoverCatalogConfigURLs(ctx context.Context, l log.Logger, opts *options.TerragruntOptions, urlCh chan<- string) error {
_, pctx := configbridge.NewParsingContext(ctx, l, opts)

status(fmt.Sprintf("Found %d module(s), loading catalog...", len(svc.Modules())))
catalogCfg, err := config.ReadCatalogConfig(ctx, l, pctx)
if err != nil {
l.Debugf("No catalog config found: %v", err)
return nil
}

if catalogCfg == nil {
return nil
}

for _, u := range catalogCfg.URLs {
urlCh <- u
}

return nil
}

// discoverSourceFileURLs walks terragrunt.hcl files, extracts
// terraform.source URLs, and sends each repo URL to urlCh.
func discoverSourceFileURLs(ctx context.Context, l log.Logger, opts *options.TerragruntOptions, urlCh chan<- string) error {
ctx, pctx := configbridge.NewParsingContext(ctx, l, opts)

urls, err := redesign.DiscoverSourceURLs(ctx, l, pctx)
if err != nil {
l.Warnf("Failed to discover source URLs: %v", err)
return nil
}

for _, u := range urls {
urlCh <- u
}

return svc, nil
})
return nil
}
4 changes: 2 additions & 2 deletions internal/cli/commands/catalog/tui/delegate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const (
selectedDescBorderForegroundColorDark = "#63C5DA"
)

func newItemDelegate(keys *delegateKeyMap) list.DefaultDelegate {
func NewItemDelegate(keys *DelegateKeyMap) list.DefaultDelegate {
d := list.NewDefaultDelegate()

d.Styles.SelectedTitle = d.Styles.SelectedTitle.
Expand All @@ -25,7 +25,7 @@ func newItemDelegate(keys *delegateKeyMap) list.DefaultDelegate {
Foreground(lipgloss.Color(selectedDescForegroundColorDark)).
BorderForeground(lipgloss.Color(selectedDescBorderForegroundColorDark))

help := []key.Binding{keys.choose, keys.scaffold}
help := []key.Binding{keys.Choose, keys.Scaffold}

d.ShortHelpFunc = func() []key.Binding {
return help
Expand Down
54 changes: 27 additions & 27 deletions internal/cli/commands/catalog/tui/keys.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import (
"charm.land/bubbles/v2/viewport"
)

// newListKeyMap returns a set of keybindings for the list view.
func newListKeyMap() list.KeyMap {
// NewListKeyMap returns a set of keybindings for the list view.
func NewListKeyMap() list.KeyMap {
return list.KeyMap{
// Browsing.
CursorUp: key.NewBinding(
Expand Down Expand Up @@ -73,51 +73,51 @@ func newListKeyMap() list.KeyMap {
}
}

type delegateKeyMap struct {
choose key.Binding
scaffold key.Binding
type DelegateKeyMap struct {
Choose key.Binding
Scaffold key.Binding
}

// Additional short help entries. This satisfies the help.KeyMap interface and
// ShortHelp returns additional short help entries. This satisfies the help.KeyMap interface and
// is entirely optional.
func (d delegateKeyMap) ShortHelp() []key.Binding { //nolint:gocritic
func (d DelegateKeyMap) ShortHelp() []key.Binding { //nolint:gocritic
return []key.Binding{
d.choose,
d.scaffold,
d.Choose,
d.Scaffold,
}
}

// Additional full help entries. This satisfies the help.KeyMap interface and
// FullHelp returns additional full help entries. This satisfies the help.KeyMap interface and
// is entirely optional.
func (d delegateKeyMap) FullHelp() [][]key.Binding { //nolint:gocritic
func (d DelegateKeyMap) FullHelp() [][]key.Binding { //nolint:gocritic
return [][]key.Binding{
{
d.choose,
d.scaffold,
d.Choose,
d.Scaffold,
},
}
}

// newDelegateKeyMap returns a set of keybindings.
func newDelegateKeyMap() *delegateKeyMap {
return &delegateKeyMap{
choose: key.NewBinding(
// NewDelegateKeyMap returns a set of keybindings.
func NewDelegateKeyMap() *DelegateKeyMap {
return &DelegateKeyMap{
Choose: key.NewBinding(
key.WithKeys("enter", "ctrl-j"),
key.WithHelp("enter/ctrl-j", "choose"),
),
scaffold: key.NewBinding(
Scaffold: key.NewBinding(
key.WithKeys("S", "s"),
key.WithHelp("S", "Scaffold"),
),
}
}

// pagerKeyMap returns a set of keybindings for the pager. It satisfies to the
// PagerKeyMap returns a set of keybindings for the pager. It satisfies to the
// help.KeyMap interface, which is used to render the menu.
type pagerKeyMap struct {
type PagerKeyMap struct {
viewport.KeyMap

help help.Model
HelpModel help.Model

// Button navigation
Navigation key.Binding
Expand All @@ -143,7 +143,7 @@ type pagerKeyMap struct {

// ShortHelp returns keybindings to be shown in the mini help view. It's part
// of the key.Map interface.
func (keys pagerKeyMap) ShortHelp() []key.Binding { //nolint:gocritic
func (keys PagerKeyMap) ShortHelp() []key.Binding { //nolint:gocritic
return []key.Binding{
keys.Up,
keys.Down,
Expand All @@ -158,17 +158,17 @@ func (keys pagerKeyMap) ShortHelp() []key.Binding { //nolint:gocritic

// FullHelp returns keybindings for the expanded help view. It's part of the
// key.Map interface.
func (keys pagerKeyMap) FullHelp() [][]key.Binding { //nolint:gocritic
func (keys PagerKeyMap) FullHelp() [][]key.Binding { //nolint:gocritic
return [][]key.Binding{
{keys.Up, keys.Down, keys.PageDown, keys.PageUp}, // first column
{keys.Navigation, keys.NavigationBack, keys.Choose, keys.Scaffold}, // second column
{keys.Help, keys.Quit, keys.ForceQuit}, // third column
}
}

// newPagerKeyMap returns a set of keybindings for the pager view.
func newPagerKeyMap() pagerKeyMap {
return pagerKeyMap{
// NewPagerKeyMap returns a set of keybindings for the pager view.
func NewPagerKeyMap() PagerKeyMap {
return PagerKeyMap{
KeyMap: viewport.KeyMap{
HalfPageUp: key.NewBinding(
key.WithDisabled(),
Expand All @@ -193,7 +193,7 @@ func newPagerKeyMap() pagerKeyMap {
key.WithHelp("h/←/pgup/alt+v", "page up"),
),
},
help: help.New(),
HelpModel: help.New(),
Navigation: key.NewBinding(
key.WithKeys("tab"),
key.WithHelp("tab", "navigation"),
Expand Down
Loading
Loading