[draft] Allow target overrides for complex-variable job tasks

.nextchanges/bundles/complex-variable-task-overrides.md

@@ -0,0 +1 @@
+* Fix target task overrides for jobs whose tasks come from a complex variable while preserving other tasks. ([#7347](https://github.com/databricks/cli/pull/7347))

acceptance/bundle/variables/issue_2436/databricks.yml

@@ -9,14 +9,14 @@ variables:
     default:
       - task_key: Task_One
         spark_python_task:
-          python_file: ./main_dummy.py
+          python_file: ./databricks.yml
         job_cluster_key: Small
       - task_key: Task_Two
         depends_on:
           - task_key: Task_One
         run_if: "ALL_SUCCESS"
         spark_python_task:
-          python_file: ./main_dummy.py
+          python_file: ./databricks.yml
         job_cluster_key: Small
 
 resources:
@@ -30,8 +30,6 @@ targets:
   dev:
     mode: development
     default: true
-    workspace:
-      host: https://...
     resources:
       jobs:
         job_one:

acceptance/bundle/variables/issue_2436/output.txt

@@ -1,107 +1,26 @@
-Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.229.0, current: [CLI_VERSION]
 
-Error: failed to perform target override for target=dev: failed to merge target=dev field=resources: cannot merge string with sequence
+>>> [CLI] bundle validate -o json
+Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.229.0, current: [CLI_VERSION]
 
 {
-  "bundle": {
-    "databricks_cli_version": "\u003e= 0.229.0",
-    "name": "testing-bundle"
-  },
-  "resources": {
-    "jobs": {
-      "job_one": {
-        "tasks": "${var.job_tasks}"
-      },
-      "job_two": {
-        "tasks": "${var.job_tasks}"
-      }
+  "job_one": [
+    {
+      "task_key": "Task_One",
+      "job_cluster_key": "Medium"
+    },
+    {
+      "task_key": "Task_Two",
+      "job_cluster_key": "Small"
     }
-  },
-  "targets": {
-    "dev": {
-      "default": true,
-      "mode": "development",
-      "resources": {
-        "jobs": {
-          "job_one": {
-            "job_clusters": [
-              {
-                "job_cluster_key": "Small",
-                "new_cluster": {
-                  "data_security_mode": "USER_ISOLATION",
-                  "node_type_id": "Standard_DS3_v2",
-                  "num_workers": 1,
-                  "runtime_engine": "STANDARD",
-                  "spark_version": "15.4.x-scala2.12"
-                }
-              },
-              {
-                "job_cluster_key": "Medium",
-                "new_cluster": {
-                  "data_security_mode": "USER_ISOLATION",
-                  "node_type_id": "Standard_DS5_v2",
-                  "num_workers": 1,
-                  "runtime_engine": "STANDARD",
-                  "spark_version": "14.3.x-scala2.12"
-                }
-              }
-            ],
-            "tasks": [
-              {
-                "job_cluster_key": "Medium",
-                "task_key": "Task_One"
-              }
-            ]
-          },
-          "job_two": {
-            "job_clusters": [
-              {
-                "job_cluster_key": "Small",
-                "new_cluster": {
-                  "data_security_mode": "USER_ISOLATION",
-                  "node_type_id": "Standard_DS3_v2",
-                  "num_workers": 2,
-                  "runtime_engine": "STANDARD",
-                  "spark_version": "15.4.x-scala2.12"
-                }
-              }
-            ]
-          }
-        }
-      },
-      "workspace": {
-        "host": "https://..."
-      }
+  ],
+  "job_two": [
+    {
+      "task_key": "Task_One",
+      "job_cluster_key": "Small"
+    },
+    {
+      "task_key": "Task_Two",
+      "job_cluster_key": "Small"
     }
-  },
-  "variables": {
-    "job_tasks": {
-      "default": [
-        {
-          "job_cluster_key": "Small",
-          "spark_python_task": {
-            "python_file": "./main_dummy.py"
-          },
-          "task_key": "Task_One"
-        },
-        {
-          "depends_on": [
-            {
-              "task_key": "Task_One"
-            }
-          ],
-          "job_cluster_key": "Small",
-          "run_if": "ALL_SUCCESS",
-          "spark_python_task": {
-            "python_file": "./main_dummy.py"
-          },
-          "task_key": "Task_Two"
-        }
-      ],
-      "description": "Common tasks",
-      "type": "complex"
-    }
-  }
+  ]
 }
-
-Exit code: 1

acceptance/bundle/variables/issue_2436/script

@@ -1 +1 @@
-$CLI bundle validate -o json
+trace $CLI bundle validate -o json | jq '{job_one: [.resources.jobs.job_one.tasks[] | {task_key, job_cluster_key}], job_two: [.resources.jobs.job_two.tasks[] | {task_key, job_cluster_key}]}'

acceptance/bundle/variables/issue_2436/test.toml

@@ -1 +1 @@
-Badness = "Fails to merge resources because targets are resolved before variables"
+Cloud = false

bundle/config/mutator/select_target_test.go

@@ -1,11 +1,14 @@
 package mutator_test
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/mutator/resourcemutator"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -41,3 +44,97 @@ func TestSelectTargetNotFound(t *testing.T) {
 	diags := bundle.Apply(t.Context(), b, mutator.SelectTarget("doesnt-exist"))
 	require.Error(t, diags.Error(), "no targets defined")
 }
+
+func TestSelectTargetMergesComplexVariableTaskOverride(t *testing.T) {
+	root, diags := config.LoadFromBytes("databricks.yml", []byte(`
+bundle:
+  name: testing-bundle
+variables:
+  job_tasks:
+    type: complex
+    default:
+      - task_key: Task_One
+        job_cluster_key: Small
+resources:
+  jobs:
+    job_one:
+      tasks: ${var.job_tasks}
+    job_two:
+      tasks: ${var.job_tasks}
+targets:
+  dev:
+    resources:
+      jobs:
+        job_one:
+          tasks:
+            - task_key: Task_One
+              job_cluster_key: Medium
+`))
+	require.NoError(t, diags.Error())
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, ".databricks", "bundle", "dev", "variable-overrides.json")
+	require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
+	require.NoError(t, os.WriteFile(path, []byte(`{"job_tasks":[{"task_key":"Task_One","job_cluster_key":"Small"},{"task_key":"Task_Two","job_cluster_key":"Small"},{"task_key":"Task_Three","job_cluster_key":"Small"}]}`), 0o600))
+
+	b := &bundle.Bundle{BundleRootPath: dir, Config: *root}
+	diags = bundle.ApplySeq(t.Context(), b,
+		mutator.SelectTarget("dev"),
+		mutator.SetVariables(),
+		mutator.ResolveVariableReferencesOnlyResources(),
+		resourcemutator.MergeJobTasks(),
+	)
+	require.NoError(t, diags.Error())
+
+	jobOne := b.Config.Resources.Jobs["job_one"].Tasks
+	require.Len(t, jobOne, 3)
+	assert.Equal(t, "Task_One", jobOne[0].TaskKey)
+	assert.Equal(t, "Medium", jobOne[0].JobClusterKey)
+	assert.ElementsMatch(t, []string{"Task_Two", "Task_Three"}, []string{jobOne[1].TaskKey, jobOne[2].TaskKey})
+
+	jobTwo := b.Config.Resources.Jobs["job_two"].Tasks
+	require.Len(t, jobTwo, 3)
+	assert.Equal(t, "Small", jobTwo[0].JobClusterKey)
+}
+
+func TestSelectTargetRewritesComplexVariableTaskOverridePath(t *testing.T) {
+	root, diags := config.LoadFromBytes("databricks.yml", []byte(`
+bundle:
+  name: testing-bundle
+workspace:
+  file_path: /Workspace/files
+variables:
+  job_tasks:
+    type: complex
+    default:
+      - task_key: Task_One
+        notebook_task:
+          notebook_path: /base
+resources:
+  jobs:
+    job_one:
+      tasks: ${var.job_tasks}
+targets:
+  dev:
+    resources:
+      jobs:
+        job_one:
+          tasks:
+            - task_key: Task_One
+              notebook_task:
+                notebook_path: /Workspace${workspace.file_path}/nb
+`))
+	require.NoError(t, diags.Error())
+
+	b := &bundle.Bundle{BundleRootPath: t.TempDir(), Config: *root}
+	diags = bundle.ApplySeq(t.Context(), b,
+		mutator.SelectTarget("dev"),
+		mutator.RewriteWorkspacePrefix(),
+		mutator.SetVariables(),
+		mutator.ResolveVariableReferencesOnlyResources(),
+		resourcemutator.MergeJobTasks(),
+	)
+	require.NoError(t, diags.Error())
+	require.Len(t, b.Config.Resources.Jobs["job_one"].Tasks, 1)
+	assert.Equal(t, "/Workspace/files/nb", b.Config.Resources.Jobs["job_one"].Tasks[0].NotebookTask.NotebookPath)
+}

bundle/config/mutator/set_variables.go

@@ -140,6 +140,10 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 			return setVariable(ctx, variable, v, name, fileDefault)
 		}))
 	})
+	if err != nil {
+		return diags.Extend(diag.FromErr(err))
+	}
 
+	err = b.Config.MergeDeferredTaskOverrides()
 	return diags.Extend(diag.FromErr(err))
 }