Add lifecycle.bind to adopt existing workspace resources at deploy time

.nextchanges/bundles/7366.md

@@ -0,0 +1 @@
+* Add `lifecycle.bind` to resources to bring an existing workspace resource under bundle management at deploy time instead of creating a new one. ([#7366](https://github.com/databricks/cli/pull/7366))

acceptance/bundle/deploy/bind/basic/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-bind-basic-UNIQUE_NAME
+
+resources:
+  experiments:
+    foo:
+      name: /Shared/test-bind-basic-UNIQUE_NAME
+      lifecycle:
+        bind:
+          id: "PLACEHOLDER_EXPERIMENT_ID"

acceptance/bundle/deploy/bind/basic/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deploy/bind/basic/output.txt

@@ -0,0 +1,42 @@
+
+>>> [CLI] bundle plan
+Bind:
+  experiments.foo (id: [NEW_EXPERIMENT_ID])
+
+Deploy:
+  No changes
+
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged, 1 to bind
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-basic-[UNIQUE_NAME]/default/files...
+Bound experiments.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged, 1 bound
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> print_state.py
+{
+ "state_version": 3,
+ "cli_version": "[CLI_VERSION]",
+ "lineage": "[UUID]",
+ "serial": 1,
+ "state": {
+  "resources.experiments.foo": {
+   "__id__": "[NEW_EXPERIMENT_ID]",
+   "state": {
+    "name": "/Shared/test-bind-basic-[UNIQUE_NAME]"
+   }
+  }
+ }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.experiments.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-basic-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/deploy/bind/basic/script

@@ -0,0 +1,25 @@
+# Bundle names carry $UNIQUE_NAME so concurrent runs on the same workspace do not share
+# a root path and its deployment lock.
+update_file.py databricks.yml 'UNIQUE_NAME' "$UNIQUE_NAME"
+
+# Create an experiment whose configuration already matches the bundle.
+NEW_EXPERIMENT_ID=$(MSYS_NO_PATHCONV=1 $CLI experiments create-experiment "/Shared/test-bind-basic-$UNIQUE_NAME" | jq -r .experiment_id)
+add_repl "$NEW_EXPERIMENT_ID" NEW_EXPERIMENT_ID
+
+# Update the databricks.yml with the actual experiment ID
+update_file.py databricks.yml 'PLACEHOLDER_EXPERIMENT_ID' "$NEW_EXPERIMENT_ID"
+
+# Run plan - should show import action
+trace $CLI bundle plan
+
+# Deploy with auto-approve
+trace $CLI bundle deploy --auto-approve
+
+# Plan again - should show no changes (skip)
+trace $CLI bundle plan
+
+# Verify state file contains the imported ID
+trace print_state.py | contains.py "$NEW_EXPERIMENT_ID"
+
+# Cleanup: destroy is allowed even with an active bind block
+trace $CLI bundle destroy --auto-approve

acceptance/bundle/deploy/bind/bind-and-resize/databricks.yml.tmpl

@@ -0,0 +1,20 @@
+bundle:
+  name: test-bind-resize
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+sync:
+  exclude:
+    - LOG.*
+
+resources:
+  clusters:
+    test_cluster:
+      cluster_name: test-cluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 3
+      lifecycle:
+        bind:
+          id: "PLACEHOLDER_CLUSTER_ID"

acceptance/bundle/deploy/bind/bind-and-resize/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deploy/bind/bind-and-resize/output.txt

@@ -0,0 +1,46 @@
+
+=== Existing cluster has 2 workers; bundle configuration requests 3
+>>> [CLI] clusters get [CLUSTER_ID]
+json.num_workers = 2;
+
+>>> [CLI] bundle plan
+Bind:
+  clusters.test_cluster (id: [CLUSTER_ID])
+
+Deploy:
+  resize clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged, 1 to bind
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Bound clusters.test_cluster
+Resized clusters.test_cluster
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged, 1 bound
+
+>>> print_requests.py //clusters/resize //clusters/edit
+{
+  "method": "POST",
+  "path": "/api/2.1/clusters/resize",
+  "body": {
+    "cluster_id": "[CLUSTER_ID]",
+    "num_workers": 3
+  }
+}
+
+=== Cluster should have new num_workers
+
+>>> [CLI] clusters get [CLUSTER_ID]
+{
+  "cluster_name": "test-cluster-[UNIQUE_NAME]",
+  "num_workers": 3
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.clusters.test_cluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted

acceptance/bundle/deploy/bind/bind-and-resize/script

@@ -0,0 +1,45 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+# The shared AWS pool uses Spot in a fixed AZ and can exhaust capacity before
+# this test reaches bind planning. Use on-demand capacity with automatic AZ selection.
+COMPUTE=$(jq -n --arg pool "$TEST_INSTANCE_POOL_ID" '{instance_pool_id: $pool}')
+if [[ "$CLOUD_ENV_BASE" == aws ]]; then
+    COMPUTE=$(jq -n --arg node "$NODE_TYPE_ID" '{node_type_id: $node, aws_attributes: {availability: "ON_DEMAND", zone_id: "auto"}}')
+    update_file.py databricks.yml "instance_pool_id: $TEST_INSTANCE_POOL_ID" "node_type_id: $NODE_TYPE_ID
+      aws_attributes:
+        availability: ON_DEMAND
+        zone_id: auto"
+fi
+jq -n --arg name "test-cluster-$UNIQUE_NAME" --arg spark "$DEFAULT_SPARK_VERSION" --argjson compute "$COMPUTE" \
+    '$compute + {cluster_name: $name, spark_version: $spark, num_workers: 2, autotermination_minutes: 60}' > LOG.create.json
+CLUSTER_ID=$($CLI clusters create --no-wait --json @LOG.create.json | jq -r .cluster_id)
+add_repl "$CLUSTER_ID" CLUSTER_ID
+update_file.py databricks.yml 'PLACEHOLDER_CLUSTER_ID' "$CLUSTER_ID"
+
+cleanup() {
+    if [[ -n "$CLUSTER_ID" ]]; then
+        $CLI clusters permanent-delete "$CLUSTER_ID" &> LOG.cluster-cleanup
+    fi
+    rm -f "$OUT_REQUESTS"
+}
+trap cleanup EXIT
+
+RETRY_MAX_ATTEMPTS=120 RETRY_INTERVAL_MS=10000 retry --until '"state": "RUNNING"' $CLI clusters get "$CLUSTER_ID" > LOG.wait
+
+title "Existing cluster has 2 workers; bundle configuration requests 3"
+trace $CLI clusters get "$CLUSTER_ID" | gron.py | grep '^json.num_workers = '
+
+# Plan: config differs only in num_workers, so the bound running cluster is
+# planned as bind_and_update backed by a resize.
+trace $CLI bundle plan
+
+# Deploy: bind_and_update must go through the resize API, not clusters/edit.
+trace $CLI bundle deploy --auto-approve
+trace print_requests.py //clusters/resize //clusters/edit
+
+title "Cluster should have new num_workers\n"
+trace $CLI clusters get "$CLUSTER_ID" | jq '{cluster_name,num_workers}'
+
+# Cleanup
+trace $CLI bundle destroy --auto-approve
+CLUSTER_ID=""

acceptance/bundle/deploy/bind/bind-and-resize/test.toml

@@ -0,0 +1,3 @@
+RecordRequests = true
+
+Ignore = [".databricks", "databricks.yml"]

acceptance/bundle/deploy/bind/bind-and-update/databricks.yml

@@ -0,0 +1,19 @@
+bundle:
+  name: test-bind-update-UNIQUE_NAME
+
+resources:
+  jobs:
+    foo:
+      name: updated-job-name
+      environments:
+        - environment_key: default
+          spec:
+            client: "1"
+      tasks:
+        - task_key: my_task
+          environment_key: default
+          spark_python_task:
+            python_file: ./hello.py
+      lifecycle:
+        bind:
+          id: "PLACEHOLDER_JOB_ID"

acceptance/bundle/deploy/bind/bind-and-update/hello.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/deploy/bind/bind-and-update/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deploy/bind/bind-and-update/output.txt

@@ -0,0 +1,41 @@
+
+>>> [CLI] bundle plan
+Bind:
+  jobs.foo (id: [NEW_JOB_ID])
+
+Deploy:
+  update jobs.foo
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged, 1 to bind
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-update-[UNIQUE_NAME]/default/files...
+Bound jobs.foo
+Updated jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged, 1 bound
+
+>>> [CLI] jobs get [NEW_JOB_ID]
+updated-job-name
+
+>>> [CLI] bundle plan
+update jobs.foo
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-update-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] jobs get [NEW_JOB_ID]
+second-update-name
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-update-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/deploy/bind/bind-and-update/script

@@ -0,0 +1,30 @@
+update_file.py databricks.yml 'UNIQUE_NAME' "$UNIQUE_NAME"
+
+# Create a job in the workspace with a different name
+NEW_JOB_ID=$($CLI jobs create --json '{"name": "original-job-name", "environments": [{"environment_key": "default", "spec": {"client": "1"}}], "tasks": [{"task_key": "my_task", "environment_key": "default", "spark_python_task": {"python_file": "/Workspace/test.py"}}]}' | jq -r .job_id)
+add_repl.py $NEW_JOB_ID NEW_JOB_ID
+
+# Update the databricks.yml with the actual job ID
+update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$NEW_JOB_ID"
+
+# Run plan - should show bind action (name differs from config)
+trace $CLI bundle plan
+
+# Deploy with auto-approve
+trace $CLI bundle deploy --auto-approve
+
+# Verify the job was updated
+trace $CLI jobs get $NEW_JOB_ID | jq -r .settings.name
+
+# Now update the job name again in the config and deploy again.
+# This time the action should be "update", not "bind", since the resource
+# is already bound in state.
+update_file.py databricks.yml 'updated-job-name' 'second-update-name'
+trace $CLI bundle plan
+trace $CLI bundle deploy --auto-approve
+
+# Verify the job was updated with the second name
+trace $CLI jobs get $NEW_JOB_ID | jq -r .settings.name
+
+# Cleanup: destroy is allowed even with an active bind block
+trace $CLI bundle destroy --auto-approve

acceptance/bundle/deploy/bind/block-removed/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+  name: test-bind-block-removed-UNIQUE_NAME
+
+resources:
+  pipelines:
+    foo:
+      name: test-pipeline
+      storage: /old/storage/path
+      libraries:
+        - notebook:
+            path: ./nb.sql
+      lifecycle:
+        bind:
+          id: "PLACEHOLDER_PIPELINE_ID"

acceptance/bundle/deploy/bind/block-removed/databricks_unbound.yml

@@ -0,0 +1,11 @@
+bundle:
+  name: test-bind-block-removed-UNIQUE_NAME
+
+resources:
+  pipelines:
+    foo:
+      name: test-pipeline
+      storage: /old/storage/path
+      libraries:
+        - notebook:
+            path: ./nb.sql

acceptance/bundle/deploy/bind/block-removed/nb.sql

@@ -0,0 +1,2 @@
+-- Databricks notebook source
+select 1

acceptance/bundle/deploy/bind/block-removed/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deploy/bind/block-removed/output.txt

@@ -0,0 +1,37 @@
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-block-removed-[UNIQUE_NAME]/default/files...
+Bound pipelines.foo
+Updated pipelines.foo
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged, 1 bound
+
+=== Removing only the bind block is a no-op
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-block-removed-[UNIQUE_NAME]/default/files...
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+
+>>> print_state.py
+json.state.resources.pipelines.foo.__id__ = "[NEW_PIPELINE_ID]";
+
+=== Without the bind block, a storage change plans a recreate
+>>> [CLI] bundle plan
+recreate pipelines.foo
+
+Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.pipelines.foo
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
+  delete resources.pipelines.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-block-removed-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/deploy/bind/block-removed/script

@@ -0,0 +1,22 @@
+update_file.py databricks.yml 'UNIQUE_NAME' "$UNIQUE_NAME"
+update_file.py databricks_unbound.yml 'UNIQUE_NAME' "$UNIQUE_NAME"
+
+# Create a pipeline whose storage matches the config and bind it
+NEW_PIPELINE_ID=$($CLI pipelines create --json '{"name": "test-pipeline", "storage": "/old/storage/path", "allow_duplicate_names": true, "libraries": [{"notebook": {"path": "/Workspace/test"}}]}' | jq -r .pipeline_id)
+add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID
+update_file.py databricks.yml 'PLACEHOLDER_PIPELINE_ID' "$NEW_PIPELINE_ID"
+
+trace $CLI bundle deploy --auto-approve
+
+title "Removing only the bind block is a no-op"
+cp databricks_unbound.yml databricks.yml
+trace $CLI bundle plan
+trace $CLI bundle deploy --auto-approve
+trace print_state.py | gron.py | grep __id__ | contains.py "$NEW_PIPELINE_ID"
+
+title "Without the bind block, a storage change plans a recreate"
+update_file.py databricks.yml "storage: /old/storage/path" "storage: /new/storage/path"
+trace $CLI bundle plan
+
+# Cleanup
+trace $CLI bundle destroy --auto-approve

acceptance/bundle/deploy/bind/block-removed/test.toml

@@ -0,0 +1 @@
+Ignore = [".databricks", "databricks_unbound.yml"]