WIP build: disable inlining for nodeup builds

Makefile

@@ -83,6 +83,9 @@ CGO_ENABLED=0
 export CGO_ENABLED
 BUILDFLAGS=-trimpath -buildvcs=false
 
+ifndef DEBUGGABLE
+NODEUP_GCFLAGS?=-gcflags=all=-l
+endif
 
 # Go exports:
 LDFLAGS := -ldflags=all=
@@ -210,7 +213,7 @@ crossbuild: crossbuild-kops-linux-amd64 crossbuild-kops-linux-arm64 crossbuild-k
 .PHONY: nodeup-amd64 nodeup-arm64
 nodeup-amd64 nodeup-arm64: nodeup-%:
 	mkdir -p ${DIST}/linux/$*
-	GOOS=linux GOARCH=$* go build ${GCFLAGS} ${BUILDFLAGS} ${EXTRA_BUILDFLAGS} -o ${DIST}/linux/$*/nodeup ${LDFLAGS}"${EXTRA_LDFLAGS} -X k8s.io/kops.Version=${VERSION} -X k8s.io/kops.GitVersion=${GITSHA}" k8s.io/kops/cmd/nodeup
+	GOOS=linux GOARCH=$* go build ${GCFLAGS} ${NODEUP_GCFLAGS} ${BUILDFLAGS} ${EXTRA_BUILDFLAGS} -o ${DIST}/linux/$*/nodeup ${LDFLAGS}"${EXTRA_LDFLAGS} -X k8s.io/kops.Version=${VERSION} -X k8s.io/kops.GitVersion=${GITSHA}" k8s.io/kops/cmd/nodeup
 
 .PHONY: nodeup
 nodeup: nodeup-amd64
@@ -295,7 +298,7 @@ push-aws-run-amd64 push-aws-run-arm64: push-aws-run-%: push-%
 
 .PHONY: ${NODEUP}
 ${NODEUP}:
-	go build ${GCFLAGS} ${BUILDFLAGS} ${EXTRA_BUILDFLAGS} ${LDFLAGS}"${EXTRA_LDFLAGS} -X k8s.io/kops.Version=${VERSION} -X k8s.io/kops.GitVersion=${GITSHA}" -o $@ k8s.io/kops/cmd/nodeup
+	go build ${GCFLAGS} ${NODEUP_GCFLAGS} ${BUILDFLAGS} ${EXTRA_BUILDFLAGS} ${LDFLAGS}"${EXTRA_LDFLAGS} -X k8s.io/kops.Version=${VERSION} -X k8s.io/kops.GitVersion=${GITSHA}" -o $@ k8s.io/kops/cmd/nodeup
 
 .PHONY: dns-controller-push
 dns-controller-push: ko-dns-controller-push

cmd/nodeup/main.go

@@ -43,7 +43,7 @@ func main() {
 
 	var flagConf, flagCacheDir, gitVersion string
 	var flagRetries int
-	var dryrun, installSystemdUnit bool
+	var dryrun, installSystemdUnit, logMemory bool
 	target := "direct"
 
 	if kops.GitVersion != "" {
@@ -56,6 +56,7 @@ func main() {
 	flag.BoolVar(&dryrun, "dryrun", false, "Don't create cloud resources; just show what would be done")
 	flag.StringVar(&target, "target", target, "Target - direct, dryrun")
 	flag.BoolVar(&installSystemdUnit, "install-systemd-unit", installSystemdUnit, "If true, will install a systemd unit instead of running directly")
+	flag.BoolVar(&logMemory, "log-memory", true, "Log Go and cgroup memory statistics during nodeup")
 
 	if dryrun {
 		target = "dryrun"
@@ -118,6 +119,7 @@ func main() {
 				ConfigLocation: flagConf,
 				Target:         target,
 				CacheDir:       flagCacheDir,
+				LogMemory:      logMemory,
 			}
 			err = cmd.Run(os.Stdout)
 			if err == nil {

upup/pkg/fi/nodeup/command.go

@@ -77,12 +77,16 @@ const MaxTaskDuration = 365 * 24 * time.Hour
 type NodeUpCommand struct {
 	CacheDir       string
 	ConfigLocation string
+	LogMemory      bool
 	Target         string
 }
 
 // Run is responsible for perform the nodeup process
 func (c *NodeUpCommand) Run(out io.Writer) error {
 	ctx := context.Background()
+	memoryLogger := newNodeupMemoryLogger(c.LogMemory)
+	memoryLogger.Log("start", nil)
+	defer memoryLogger.Log("exit", nil)
 
 	var bootConfig nodeup.BootConfig
 	if c.ConfigLocation != "" {
@@ -102,6 +106,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 	if c.CacheDir == "" {
 		return fmt.Errorf("CacheDir is required")
 	}
+	memoryLogger.Log("boot-config-loaded", nil)
 
 	region, err := getRegion(ctx, &bootConfig)
 	if err != nil {
@@ -170,6 +175,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 	if err != nil {
 		return err
 	}
+	memoryLogger.Log("nodeup-config-loaded", nil)
 
 	architecture, err := architectures.FindArchitecture()
 	if err != nil {
@@ -189,6 +195,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 			return fmt.Errorf("error adding asset %q: %v", asset, err)
 		}
 	}
+	memoryLogger.Log("assets-loaded", nil)
 
 	var cloud fi.Cloud
 
@@ -246,6 +253,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 	if err := modelContext.Init(); err != nil {
 		return err
 	}
+	memoryLogger.Log("model-context-initialized", nil)
 
 	switch bootConfig.CloudProvider {
 	case api.CloudProviderAWS:
@@ -290,10 +298,12 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 			modelContext.GPUVendor = architectures.GPUVendorNvidia
 		}
 	}
+	memoryLogger.Log("node-metadata-loaded", nil)
 
 	if err := loadKernelModules(modelContext, distribution); err != nil {
 		return err
 	}
+	memoryLogger.Log("kernel-modules-loaded", nil)
 
 	loader := &Loader{}
 	loader.Builders = append(loader.Builders, &model.DiscoveryService{NodeupModelContext: modelContext})
@@ -350,6 +360,8 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 		}
 	}
 	// Protokube load image task is in ProtokubeBuilder
+	taskSummary := summarizeNodeupMemoryTasks(taskMap)
+	memoryLogger.Log("task-graph-built", &taskSummary)
 
 	var target fi.NodeupTarget
 
@@ -370,24 +382,33 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 	if err != nil {
 		klog.Exitf("error building context: %v", err)
 	}
+	memoryLogger.Log("context-built", &taskSummary)
 
 	var options fi.RunTasksOptions
 	options.InitDefaults()
 
+	memoryLogger.Log("before-run-tasks", &taskSummary)
+	stopMemoryLogging := memoryLogger.StartPeriodic("run-tasks", &taskSummary)
 	err = context.RunTasks(options)
+	stopMemoryLogging()
 	if err != nil {
+		memoryLogger.Log("run-tasks-error", &taskSummary)
 		klog.Exitf("error running tasks: %v", err)
 	}
+	memoryLogger.Log("after-run-tasks", &taskSummary)
 
 	err = target.Finish(taskMap)
 	if err != nil {
+		memoryLogger.Log("target-finish-error", &taskSummary)
 		klog.Exitf("error closing target: %v", err)
 	}
+	memoryLogger.Log("after-target-finish", &taskSummary)
 
 	if nodeupConfig.EnableLifecycleHook {
 		if bootConfig.CloudProvider == api.CloudProviderAWS {
 			err := completeWarmingLifecycleAction(ctx, cloud.(awsup.AWSCloud), modelContext)
 			if err != nil {
+				memoryLogger.Log("lifecycle-hook-error", &taskSummary)
 				return fmt.Errorf("failed to complete lifecylce action: %w", err)
 			}
 		}