Fail the invariant coverage test when a config template is not in the matrix

acceptance/invariant_test.go

@@ -7,14 +7,18 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/databricks/cli/acceptance/internal"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/dyn/yamlloader"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-const invariantConfigsDir = "bundle/invariant/configs"
+const (
+	invariantConfigsDir = "bundle/invariant/configs"
+	invariantTestConfig = "bundle/invariant/test.toml"
+)
 
 // LackingInvariantTest lists keys from config.ResourcesTypes that knowingly lack
 // a covering config in invariantConfigsDir. Keys match the ResourcesTypes
@@ -26,6 +30,17 @@ var LackingInvariantTest = map[string]bool{
 	"internal_immutable_snapshots": true,
 }
 
+// UnlistedInvariantConfigs lists templates in invariantConfigsDir that are not in
+// EnvMatrix["INPUT_CONFIG"] of invariantTestConfig, so the invariant suite never runs them.
+// The list may only shrink: add a template to INPUT_CONFIG and remove it here to run it.
+var UnlistedInvariantConfigs = map[string]bool{
+	"app_lifecycle_ref.yml.tmpl":          true,
+	"materialization_set.yml.tmpl":        true,
+	"mcp_service.yml.tmpl":                true,
+	"model_provider_service.yml.tmpl":     true,
+	"postgres_snapshot_schedule.yml.tmpl": true,
+}
+
 // TestInvariantConfigsCoverage ensures that the invariant test configs in
 // bundle/invariant/configs cover every bundle resource type, and that resource
 // types supporting permissions or grants have at least one config exercising them.
@@ -69,6 +84,42 @@ func TestInvariantConfigsCoverage(t *testing.T) {
 	}
 }
 
+// TestInvariantConfigsListed ensures every template in invariantConfigsDir is run by the
+// invariant matrix unless allowlisted in UnlistedInvariantConfigs, and that every matrix
+// entry names an existing template. A template missing from the matrix is silently never run.
+func TestInvariantConfigsListed(t *testing.T) {
+	matrix := internal.DoLoadConfig(t, invariantTestConfig).EnvMatrix["INPUT_CONFIG"]
+	listed := make(map[string]bool, len(matrix))
+	for _, name := range matrix {
+		listed[name] = true
+	}
+
+	entries, err := os.ReadDir(invariantConfigsDir)
+	require.NoError(t, err)
+
+	existing := map[string]bool{}
+	for _, entry := range entries {
+		name := entry.Name()
+		if entry.IsDir() || !strings.HasSuffix(name, ".yml.tmpl") {
+			continue
+		}
+		existing[name] = true
+
+		if UnlistedInvariantConfigs[name] {
+			assert.False(t, listed[name],
+				"%q is in INPUT_CONFIG of %s; remove it from UnlistedInvariantConfigs", name, invariantTestConfig)
+			continue
+		}
+		assert.True(t, listed[name],
+			"%q in %s is not in INPUT_CONFIG of %s, so it never runs; add it or allowlist it in UnlistedInvariantConfigs",
+			name, invariantConfigsDir, invariantTestConfig)
+	}
+
+	for _, name := range matrix {
+		assert.True(t, existing[name], "INPUT_CONFIG entry %q has no template in %s", name, invariantConfigsDir)
+	}
+}
+
 // scanInvariantConfigs parses every config in the invariant configs directory and
 // returns the set of resource groups present, the groups with at least one resource
 // carrying permissions, and the groups with at least one resource carrying grants.