WIP Change from coveralls to codev

.github/workflows/coverage-quality.yml

@@ -0,0 +1,32 @@
+name: Coverage Quality Gate
+
+on:
+  pull_request:
+    paths-ignore: ['**/*.md', 'docs/**', 'hack/**', 'designs/**']
+
+jobs:
+  coverage-check:
+    name: Ensure Coverage Does Not Decrease
+    runs-on: ubuntu-latest
+    steps:
+      - name: Codecov Check
+        run: echo "Coverage quality is checked by Codecov status checks in the PR"
+
+      - name: Coverage Info
+        run: |
+          echo "Coverage is tracked by Codecov"
+          echo "View detailed coverage at: https://codecov.io/gh/kubernetes-sigs/kubebuilder"
+          echo "Coverage status will appear as a check on this PR"
+          echo ""
+          echo "Coverage is collected from:"
+          echo "  - Unit tests (coverage.yml)"
+          echo "  - Integration tests (coverage.yml)"
+          echo "  - E2E cluster tests (e2e-coverage.yml)"
+          echo ""
+          echo "Files excluded from coverage:"
+          echo "  - Test files (*_test.go)"
+          echo "  - Template files (scaffolds/internal/templates/**)"
+          echo "  - Documentation (docs/**, **/*.md)"
+          echo "  - Configuration files (**/*.yaml, **/*.yml)"
+          echo "  - Scripts (hack/**, **/*.sh)"
+

.github/workflows/coverage.yml

@@ -2,12 +2,13 @@ name: Coverage
 
 on:
   push:
-    paths-ignore: ['**/*.md']
+    paths-ignore: ['**/*.md', '**/*.yaml', '**/*.yml', 'docs/**', 'hack/**', 'designs/**']
   pull_request:
-    paths-ignore: ['**/*.md']
+    paths-ignore: ['**/*.md', '**/*.yaml', '**/*.yml', 'docs/**', 'hack/**', 'designs/**']
 
 jobs:
-  coverage:
+  unit-integration-coverage:
+    name: Unit & Integration Tests Coverage
     runs-on: ubuntu-latest
     if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository)
     steps:
@@ -25,10 +26,19 @@ jobs:
       - name: Install Kubebuilder
         run: make install
 
-      - name: Run tests with coverage
+      - name: Run unit and integration tests with coverage
         run: make test-coverage
+        # Note: Integration tests scaffold projects and test file generation
+        # They do NOT require a Kubernetes cluster (no kind/kubectl needed)
 
-      - name: Upload coverage to Coveralls
-        uses: shogo82148/actions-goveralls@v1
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
         with:
-          path-to-profile: coverage-all.out
+          files: ./coverage-all.out
+          flags: unit,integration
+          name: unit-integration
+          fail_ci_if_error: false
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+        # Note: For public repos, token is optional but recommended
+        # If CODECOV_TOKEN is not set, it will still work for public repos

.github/workflows/e2e-coverage.yml

@@ -0,0 +1,54 @@
+name: E2E Coverage
+
+on:
+  push:
+    branches:
+      - master
+      - main
+    paths-ignore: ['**/*.md', 'docs/**', 'hack/**', 'designs/**']
+  pull_request:
+    paths-ignore: ['**/*.md', 'docs/**', 'hack/**', 'designs/**']
+
+jobs:
+  e2e-coverage:
+    name: E2E Cluster Tests with Coverage
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6.0.1
+
+      - name: Setup Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+
+      - name: Install Kubebuilder
+        run: make install
+
+      - name: Install kind
+        run: |
+          curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-$(go env GOARCH)
+          chmod +x ./kind
+          sudo mv ./kind /usr/local/bin/kind
+
+      - name: Verify kind installation
+        run: kind version
+
+      - name: Run E2E cluster tests with coverage
+        run: ./test/e2e/coverage.sh
+        # Note: E2E tests scaffold projects, build them, deploy to kind cluster,
+        # and verify they work end-to-end (requires Kubernetes cluster)
+
+      - name: Upload E2E coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: ./coverage-e2e.out
+          flags: e2e
+          name: e2e-cluster-tests
+          fail_ci_if_error: false
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+        # Note: For public repos, token is optional but recommended
+        # If CODECOV_TOKEN is not set, it will still work for public repos
+

CONTRIBUTING.md

@@ -63,7 +63,7 @@ Following the targets that can be used to test your changes locally.
 | make test           | Runs tests in shell (`./test.sh`)                             | yes                  |
 | make lint           | Run [golangci][golangci] lint checks                          | yes                  |
 | make lint-fix       | Run [golangci][golangci] to automatically perform fixes       | no                   |
-| make test-coverage  | Run coveralls to check the % of code covered by tests         | yes                  |
+| make test-coverage  | Run unit+integration tests with coverage (uploads to Codecov) | yes                  |
 | make check-testdata | Checks if the testdata dir is updated with the latest changes | yes                  |
 | make test-e2e-local | Runs the CI e2e tests locally                                 | no                   |
 

Makefile

@@ -157,9 +157,14 @@ test-integration: install ## Run the integration tests (requires kubebuilder bin
 	go test -race -tags=integration -timeout 30m $(TEST_PKGS)
 
 .PHONY: test-coverage
-test-coverage: ## Run unit and integration tests with coverage report
+test-coverage: ## Run unit and integration tests with coverage report (for Codecov)
 	- rm -rf *.out  # Remove all coverage files if exists
-	go test -race -failfast -tags=integration -timeout 30m -coverprofile=coverage-all.out -coverpkg="./pkg/cli/...,./pkg/config/...,./pkg/internal/...,./pkg/machinery/...,./pkg/model/...,./pkg/plugin/...,./pkg/plugins/golang/...,./pkg/plugins/external/...,./pkg/plugins/optional/grafana/...,./pkg/plugins/optional/helm/v2alpha/..." $(TEST_PKGS)
+	go test -race -failfast -tags=integration -timeout 30m -coverprofile=coverage-all.out -coverpkg="./pkg/..." $(TEST_PKGS)
+
+.PHONY: test-e2e-coverage
+test-e2e-coverage: install ## Run e2e tests with coverage (requires kind cluster)
+	- rm -rf coverage-e2e*.out
+	./test/e2e/local.sh -coverprofile=coverage-e2e.out -coverpkg="./pkg/..."
 
 .PHONY: check-testdata
 check-testdata: ## Run the script to ensure that the testdata is updated

README.md

@@ -1,7 +1,7 @@
 [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/kubernetes-sigs/kubebuilder/badge)](https://scorecard.dev/viewer/?uri=github.com/kubernetes-sigs/kubebuilder)
 [![Lint](https://github.com/kubernetes-sigs/kubebuilder/actions/workflows/lint.yml/badge.svg)](https://github.com/kubernetes-sigs/kubebuilder/actions/workflows/lint.yml)
 [![Go Report Card](https://goreportcard.com/badge/sigs.k8s.io/kubebuilder)](https://goreportcard.com/report/sigs.k8s.io/kubebuilder)
-[![Coverage Status](https://coveralls.io/repos/github/kubernetes-sigs/kubebuilder/badge.svg?branch=master)](https://coveralls.io/github/kubernetes-sigs/kubebuilder?branch=master)
+[![codecov](https://codecov.io/gh/kubernetes-sigs/kubebuilder/branch/master/graph/badge.svg)](https://codecov.io/gh/kubernetes-sigs/kubebuilder)
 [![Latest release](https://img.shields.io/github/v/release/kubernetes-sigs/kubebuilder)](https://github.com/kubernetes-sigs/kubebuilder/releases)
 [![Go Reference](https://pkg.go.dev/badge/sigs.k8s.io/kubebuilder/v4.svg)](https://pkg.go.dev/sigs.k8s.io/kubebuilder/v4)
 

codecov.yml

@@ -0,0 +1,76 @@
+# Codecov configuration
+# See: https://docs.codecov.com/docs/codecov-yaml
+
+coverage:
+  precision: 2
+  round: down
+  range: "60...90"
+
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%  # Allow 1% decrease
+        base: auto
+        flags:
+          - unit
+          - integration
+          - e2e
+    patch:
+      default:
+        target: auto
+        threshold: 5%  # New code should have reasonable coverage
+
+# Flags allow us to see coverage breakdown by test type
+# - unit: Unit tests (no external dependencies)
+# - integration: Integration tests (scaffold projects, test file generation, no cluster)
+# - e2e: E2E tests (scaffold + build + deploy to kind cluster + verify)
+flags:
+  unit:
+    paths:
+      - pkg/
+    carryforward: true
+  integration:
+    paths:
+      - pkg/
+    carryforward: true
+  e2e:
+    paths:
+      - pkg/
+    carryforward: true
+
+# Ignore files/paths that shouldn't be measured
+ignore:
+  - "**/*_test.go"
+  - "**/test/**"
+  - "**/testdata/**"
+  - "hack/**"
+  - "docs/**"
+  - "**/*.md"
+  - "**/*.yaml"
+  - "**/*.yml"
+  - "**/*.json"
+  - "**/*.sh"
+  - "build/**"
+  - "designs/**"
+  - "roadmap/**"
+  - "scripts/**"
+  # Template files - these are mostly static strings
+  - "pkg/plugins/**/scaffolds/internal/templates/**"
+  - "pkg/plugins/golang/v4/scaffolds/internal/templates/**"
+  - "pkg/plugins/golang/deploy-image/**/scaffolds/internal/templates/**"
+  - "pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/**"
+  - "pkg/plugins/optional/**/scaffolds/internal/templates/**"
+
+# Comment on PRs with coverage information
+comment:
+  layout: "reach,diff,flags,tree,footer"
+  behavior: default
+  require_changes: false
+  require_base: false
+  require_head: true
+
+# GitHub checks
+github_checks:
+  annotations: true
+

test/e2e/coverage.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script runs e2e tests with coverage collection for Codecov
+
+source "$(dirname "$0")/../common.sh"
+source "$(dirname "$0")/setup.sh"
+
+export KIND_CLUSTER="kind"
+create_cluster ${KIND_K8S_VERSION}
+trap delete_cluster EXIT
+
+test_cluster_with_coverage
+

test/e2e/setup.sh

@@ -87,3 +87,19 @@ function test_cluster {
   go test $(dirname "$0")/v4 $flags -timeout 30m
   go test $(dirname "$0")/helm $flags -timeout 30m
 }
+
+# Test cluster with coverage - merges coverage from all e2e test packages
+function test_cluster_with_coverage {
+  local e2e_dir=$(dirname "$0")
+
+  # Run each e2e test suite with coverage
+  go test $e2e_dir/deployimage -v -ginkgo.vv -timeout 30m -coverprofile=coverage-e2e-deployimage.out -coverpkg=./pkg/...
+  go test $e2e_dir/v4 -v -ginkgo.vv -timeout 30m -coverprofile=coverage-e2e-v4.out -coverpkg=./pkg/...
+  go test $e2e_dir/helm -v -ginkgo.vv -timeout 30m -coverprofile=coverage-e2e-helm.out -coverpkg=./pkg/...
+
+  # Merge coverage files
+  echo "mode: atomic" > coverage-e2e.out
+  tail -n +2 coverage-e2e-deployimage.out >> coverage-e2e.out 2>/dev/null || true
+  tail -n +2 coverage-e2e-v4.out >> coverage-e2e.out 2>/dev/null || true
+  tail -n +2 coverage-e2e-helm.out >> coverage-e2e.out 2>/dev/null || true
+}