From 31d2dd3c254263c0d46c22af5a123123c01c025b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 13:44:37 +0100 Subject: [PATCH 01/21] Rename schemas in place in the direct engine Changing a schema's name used to delete and recreate it. It is now an updatable ID field renamed via PATCH new_name, which keeps the schema and everything in it. UC moves the children of a renamed catalog or schema with it. A child volume (or a schema in a renamed catalog) whose catalog_name/schema_name references the parent's name was planned as a recreate, and its create then failed with RESOURCE_ALREADY_EXISTS on every deploy. The planner now re-keys such children (update_id, reason parent_renamed), and they are addressed under their new parents. The test server now moves children on catalog/schema rename and rejects duplicate volume names, matching UC. Co-authored-by: Isaac --- .nextchanges/bundles/schema-rename.md | 1 + .../bundle/lifecycle/prevent-destroy/script | 4 +- .../migrate/auto/schema-rename/output.txt | 35 +--- .../bundle/migrate/auto/schema-rename/script | 15 +- .../bundle/migrate/auto/volume-rename/script | 2 +- .../catalogs/rename/databricks.yml.tmpl | 16 ++ .../resources/catalogs/rename/out.test.toml | 2 + .../resources/catalogs/rename/output.txt | 176 ++++++++++++++++++ .../bundle/resources/catalogs/rename/script | 47 +++++ .../resources/catalogs/rename/test.toml | 7 + .../schemas/rename/databricks.yml.tmpl | 9 + .../resources/schemas/rename/out.test.toml | 2 + .../resources/schemas/rename/output.txt | 99 ++++++++++ .../bundle/resources/schemas/rename/script | 47 +++++ .../bundle/resources/schemas/rename/test.toml | 7 + .../rename_with_volume/databricks.yml.tmpl | 13 ++ .../schemas/rename_with_volume/out.test.toml | 2 + .../schemas/rename_with_volume/output.txt | 132 +++++++++++++ .../schemas/rename_with_volume/script | 48 +++++ .../schemas/rename_with_volume/test.toml | 7 + .../volumes/uppercase-name/out.deploy.txt | 2 +- bundle/deployplan/plan.go | 4 + bundle/direct/bundle_plan.go | 92 +++++++++ bundle/direct/bundle_plan_test.go | 58 ++++++ bundle/direct/dresources/configs/schemas.yml | 7 +- bundle/direct/dresources/schema.go | 36 +++- bundle/direct/dresources/volume.go | 6 +- libs/testserver/catalogs.go | 23 +++ libs/testserver/schemas.go | 26 ++- libs/testserver/volumes.go | 10 + 30 files changed, 881 insertions(+), 54 deletions(-) create mode 100644 .nextchanges/bundles/schema-rename.md create mode 100644 acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/rename/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/rename/output.txt create mode 100644 acceptance/bundle/resources/catalogs/rename/script create mode 100644 acceptance/bundle/resources/catalogs/rename/test.toml create mode 100644 acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename/script create mode 100644 acceptance/bundle/resources/schemas/rename/test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/script create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/test.toml diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md new file mode 100644 index 00000000000..bc30b0568b4 --- /dev/null +++ b/.nextchanges/bundles/schema-rename.md @@ -0,0 +1 @@ +* Rename a schema in place when its `name` changes, instead of deleting and recreating it (direct engine). Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#PR](https://github.com/databricks/cli/pull/PR)) diff --git a/acceptance/bundle/lifecycle/prevent-destroy/script b/acceptance/bundle/lifecycle/prevent-destroy/script index a2c90cea5b5..94c074cfc0b 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/script +++ b/acceptance/bundle/lifecycle/prevent-destroy/script @@ -12,8 +12,8 @@ update_file.py resources/pipeline.yml ' catalog: main' ' storage: "dbf trace errcode $CLI bundle plan >>out.txt 2>&1 trace musterr $CLI bundle deploy -qq >>out.txt 2>&1 -# Changing the schema name, deploy must fail because schema will be recreated -update_file.py resources/schema.yml 'name: test-schema' 'name: test-schema-new' +# Changing the schema catalog, deploy must fail because schema will be recreated +update_file.py resources/schema.yml 'catalog_name: "test-catalog"' 'catalog_name: "test-catalog-new"' trace errcode $CLI bundle plan >>out.txt 2>&1 trace musterr $CLI bundle deploy -qq >>out.txt 2>&1 diff --git a/acceptance/bundle/migrate/auto/schema-rename/output.txt b/acceptance/bundle/migrate/auto/schema-rename/output.txt index 65e060f08f9..6985c121152 100644 --- a/acceptance/bundle/migrate/auto/schema-rename/output.txt +++ b/acceptance/bundle/migrate/auto/schema-rename/output.txt @@ -2,38 +2,15 @@ === Replay the prior terraform deployment (captured from a real terraform run) >>> replay_tfstate.py terraform-requests.json terraform.tfstate -=== Rename the schema (an immutable id field). Without --auto-approve the recreate is refused ->>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy -Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. -Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.my_schema -Warn: Migration not committed, keeping Terraform state -Error: the deployment requires destructive actions, but the current console does not support prompting. -Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. -To proceed, use --auto-approve after reviewing the plan above. - -Files: 2 uploaded, 0 deleted - -Exit code: 1 - ->>> print_migration_telemetry -direct_migrate_recreate_planned true - -=== With --auto-approve the recreate is applied on direct under the new name ->>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve -Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. +=== Rename the schema (an updatable id field) and deploy on direct +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be renamed. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.my_schema Migrated 1 resource to direct deployment engine. -Recreated schemas.my_schema -Files: 0 uploaded, 0 deleted -Resources: 1 created, 0 changed, 1 deleted, 0 unchanged +Updated schemas.my_schema +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Redeploy on direct: the schema now matches config, so the plan is a clean no-op >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy diff --git a/acceptance/bundle/migrate/auto/schema-rename/script b/acceptance/bundle/migrate/auto/schema-rename/script index 2286ccca1b2..273c43e1917 100644 --- a/acceptance/bundle/migrate/auto/schema-rename/script +++ b/acceptance/bundle/migrate/auto/schema-rename/script @@ -10,17 +10,12 @@ title "Replay the prior terraform deployment (captured from a real terraform run trace replay_tfstate.py terraform-requests.json terraform.tfstate rm -f "$OUT_REQUESTS" -title "Rename the schema (an immutable id field). Without --auto-approve the recreate is refused" -# name is a provided id field, so a rename recreates. reconcileIDFields seeds the deployed -# name into the converted state, so the migration's plan sees a real recreate. Schemas are in -# the destructive-approval group, so without --auto-approve the deploy refuses the recreate -# (exit 1) and the migration is not committed - the run stays on terraform. +title "Rename the schema (an updatable id field) and deploy on direct" +# name is an updatable id field: the schema is renamed in place (UpdateWithID), so the deploy +# needs no --auto-approve. reconcileIDFields seeds the deployed name into the converted state, +# so the migration's plan sees a rename rather than snapshotting the new name as already applied. update_file.py databricks.yml "myschema-$UNIQUE_NAME" "myschema-$UNIQUE_NAME-renamed" -trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy -trace print_migration_telemetry - -title "With --auto-approve the recreate is applied on direct under the new name" -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy title "Redeploy on direct: the schema now matches config, so the plan is a clean no-op" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy diff --git a/acceptance/bundle/migrate/auto/volume-rename/script b/acceptance/bundle/migrate/auto/volume-rename/script index 5ef772365b7..b4d3abb6b3b 100644 --- a/acceptance/bundle/migrate/auto/volume-rename/script +++ b/acceptance/bundle/migrate/auto/volume-rename/script @@ -10,7 +10,7 @@ title "Replay the prior terraform deployment (captured from a real terraform run trace replay_tfstate.py terraform-requests.json terraform.tfstate title "Rename the volume (an updatable id field) and deploy on direct" -# Unlike a schema (whose name recreates), a volume name is an updatable id field: it is +# A volume name is an updatable id field: it is # renamed in place (UpdateWithID). The migration seeds the deployed name into the converted # state, so its plan sees a rename and the migration succeeds; the deploy then renames the # volume rather than the migration snapshotting the new name as already applied. Verifies the diff --git a/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/catalogs/rename/out.test.toml b/acceptance/bundle/resources/catalogs/rename/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt new file mode 100644 index 00000000000..a4a6e81a101 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -0,0 +1,176 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "test_catalog_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} + +=== Rename the catalog +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +=== UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them +>>> [CLI] bundle plan +update catalogs.catalog1 +update schemas.schema1 +update volumes.volume1 + +Plan: 0 to add, 3 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + }, + "resources.schemas.schema1": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + }, + "resources.volumes.volume1": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } +} + +=== Deploy renames the catalog and updates the schema and the volume at their new paths +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Updated schemas.schema1 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 3 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_renamed_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0 + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "body": { + "comment": "" + } +} + +=== The old catalog is gone and the schema and the volume are under the new one +old catalog is gone + +>>> [CLI] schemas get test_catalog_renamed_[UNIQUE_NAME].myschema +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +>>> [CLI] volumes read test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +=== The schema_id and volume_id are preserved, so these are the same objects +schema_id is unchanged +volume_id is unchanged + +=== The state tracks the new paths +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume" + } +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script new file mode 100644 index 00000000000..d9421c00eb8 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -0,0 +1,47 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_CATALOG="test_catalog_${UNIQUE_NAME}" +NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +SCHEMA_ID=$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id) +VOLUME_ID=$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id) + +title "Rename the catalog" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" +trace $CLI bundle plan +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values(.changes | with_entries(select(.value.action != "skip")))' + +title "Deploy renames the catalog and updates the schema and the volume at their new paths" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The old catalog is gone and the schema and the volume are under the new one\n" +# Not traced: the "not found" message differs between UC and the test server. +$CLI catalogs get "$OLD_CATALOG" &> /dev/null && echo "old catalog still exists" || echo "old catalog is gone" +trace $CLI schemas get "$NEW_CATALOG.myschema" | jq '{full_name, catalog_name}' +trace $CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq '{full_name, catalog_name}' + +title "The schema_id and volume_id are preserved, so these are the same objects\n" +[ "$($CLI schemas get "$NEW_CATALOG.myschema" | jq -r .schema_id)" = "$SCHEMA_ID" ] && echo "schema_id is unchanged" || echo "schema_id changed" +[ "$($CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq -r .volume_id)" = "$VOLUME_ID" ] && echo "volume_id is unchanged" || echo "volume_id changed" + +title "The state tracks the new paths" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/catalogs/rename/test.toml b/acceptance/bundle/resources/catalogs/rename/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl new file mode 100644 index 00000000000..080c703a351 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + comment: COMMENT1 diff --git a/acceptance/bundle/resources/schemas/rename/out.test.toml b/acceptance/bundle/resources/schemas/rename/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt new file mode 100644 index 00000000000..a213c84394b --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -0,0 +1,99 @@ + +=== Deploy the schema +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "comment": "COMMENT1", + "name": "test-schema-[UNIQUE_NAME]" + } +} + +=== Rename the schema +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== The plan renames the schema in place instead of recreating it +>>> [CLI] bundle plan +update schemas.schema1 + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan --output json +{ + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } +} + +=== Deploy renames the schema without a deletion prompt +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "COMMENT1", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== The schema is only reachable under the new name +>>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] +Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. + +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "comment": "COMMENT1" +} + +=== The schema_id is preserved, so this is the same schema +schema_id is unchanged + +=== The state and summary track the new name +>>> [CLI] bundle summary --output json +{ + "id": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script new file mode 100644 index 00000000000..24dacb067dc --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/script @@ -0,0 +1,47 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy the schema" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +SCHEMA_ID=$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id) + +title "Rename the schema" +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "The plan renames the schema in place instead of recreating it" +trace $CLI bundle plan +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' | normalize_uc_payload.py + +title "Deploy renames the schema without a deletion prompt" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The schema is only reachable under the new name" +trace musterr $CLI schemas get "$OLD_SCHEMA" +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, name, comment}' + +title "The schema_id is preserved, so this is the same schema\n" +if [ "$($CLI schemas get "$NEW_SCHEMA" | jq -r .schema_id)" = "$SCHEMA_ID" ]; then + echo "schema_id is unchanged" +else + echo "schema_id changed" +fi + +title "The state and summary track the new name" +trace $CLI bundle summary --output json | jq '.resources.schemas.schema1 | {id, name}' + +title "Redeploy is a no-op" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/rename/test.toml b/acceptance/bundle/resources/schemas/rename/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt new file mode 100644 index 00000000000..17f9535e737 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -0,0 +1,132 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the schema +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it +>>> [CLI] bundle plan +update schemas.schema1 +update volumes.volume1 + +Plan: 0 to add, 2 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + }, + "resources.volumes.volume1": { + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + } +} + +=== Deploy renames the schema and updates the volume at its new path +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "" + } +} + +=== The old schema is gone and the volume is under the new one +>>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] +Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]" +} + +=== The volume_id is preserved, so this is the same volume +volume_id is unchanged + +=== The state tracks the new path +>>> [CLI] bundle summary --output json +"main.test-schema-renamed-[UNIQUE_NAME].myvolume" + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script new file mode 100644 index 00000000000..478233c03aa --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -0,0 +1,48 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +OLD_VOLUME="$OLD_SCHEMA.myvolume" +NEW_VOLUME="main.test-schema-renamed-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +VOLUME_ID=$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id) + +title "Rename the schema" +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" +trace $CLI bundle plan +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values(.changes | with_entries(select(.value.action != "skip")))' + +title "Deploy renames the schema and updates the volume at its new path" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The old schema is gone and the volume is under the new one" +trace musterr $CLI schemas get "$OLD_SCHEMA" +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, schema_name}' + +title "The volume_id is preserved, so this is the same volume\n" +if [ "$($CLI volumes read "$NEW_VOLUME" | jq -r .volume_id)" = "$VOLUME_ID" ]; then + echo "volume_id is unchanged" +else + echo "volume_id changed" +fi + +title "The state tracks the new path" +trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' + +title "Redeploy is a no-op" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/test.toml b/acceptance/bundle/resources/schemas/rename_with_volume/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt index bba6a759059..2b50dcb8cfa 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt @@ -33,7 +33,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "changes": { "name": { "action": "skip", - "reason": "id_field" + "reason": "id_changes" } } }, diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 8446b200b12..d53de5eee48 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -184,6 +184,10 @@ const ( // it and whatever the remote reports is not drift (e.g. cluster libraries without a libraries // section, where job runs install their task libraries cluster-wide). ReasonUnmanaged = "unmanaged" + // ReasonParentRenamed: an ID field that references the name of a parent being renamed in + // place. The backend moves the resource along with its parent, so it is re-keyed rather + // than recreated. + ReasonParentRenamed = "parent_renamed" // Special reason that results in removing this change from the plan ReasonDrop = "!drop" diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 8f8336beaa5..6042aa4f33c 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -263,6 +263,13 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return true } + // Collected before resolveReferences, which drops the references it resolves. + parentRenames, err := b.parentRenames(resourceKey) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + // Process all references in the resource using Refs map // Refs maps path inside resource to references e.g. "${resources.jobs.foo.id} ${resources.jobs.foo.name}" if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, true) { @@ -370,6 +377,12 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } + err = followParentRenames(adapter, entry.Changes, parentRenames) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) + return false + } + if remoteState == nil { // Even if local action is "recreate" which is higher than "create", we should still pick "create" here // because we know remote does not exist. @@ -666,6 +679,85 @@ func classifyIDField(cfg *dresources.ResourceLifecycleConfig, path *structpath.P return deployplan.Undefined, "", false } +// parentRenames returns the fields of resourceKey whose whole value is a reference to a +// field that the referenced resource renames in this plan (an update_id change), keyed by +// field path, with the referenced change as the value. +func (b *DeploymentBundle) parentRenames(resourceKey string) (map[string]*deployplan.ChangeDesc, error) { + sv, ok := b.StateCache.Load(resourceKey) + if !ok { + return nil, fmt.Errorf("internal error: no state cache entry found for %q", resourceKey) + } + + result := make(map[string]*deployplan.ChangeDesc) + for fieldPath, refString := range sv.Refs { + ref, ok := dynvar.NewRef(dyn.V(refString)) + if !ok || !ref.IsPure() { + continue + } + targetPath, err := structpath.ParsePath(ref.References()[0]) + if err != nil { + return nil, fmt.Errorf("cannot parse reference %q: %w", refString, err) + } + targetKey, targetField := splitResourcePath(targetPath) + change, err := b.renameChange(targetKey, targetField.String()) + if err != nil { + return nil, err + } + if change != nil { + result[fieldPath] = change + } + } + return result, nil +} + +// renameChange returns the change to field of resourceKey if it is part of an update_id, nil otherwise. +func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan.ChangeDesc, error) { + entry, err := b.Plan.ReadLockEntry(resourceKey) + if err != nil { + return nil, err + } + defer b.Plan.ReadUnlockEntry(resourceKey) + + // A missing entry is reported by resolveReferences. + if entry == nil || entry.Action != deployplan.UpdateWithID { + return nil, nil + } + change := entry.Changes[field] + if change == nil || change.Action != deployplan.UpdateWithID { + return nil, nil + } + return change, nil +} + +// followParentRenames plans an update_id instead of a recreate for an ID field that changes +// only because it references the name of a parent that is renamed in place. UC moves the +// children of a renamed catalog or schema along with it, so recreating such a child fails +// with "already exists"; its DoUpdateWithID addresses it under the new parent instead. +// +// The old values must match too: a child moved to the renamed parent from somewhere else +// still has to be recreated. +func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) error { + if !adapter.HasDoUpdateWithID() { + return nil + } + for pathString, parentChange := range parentRenames { + ch := changes[pathString] + if ch == nil || ch.Action != deployplan.Recreate || !structdiff.IsEqual(ch.Old, parentChange.Old) { + continue + } + path, err := structpath.ParsePath(pathString) + if err != nil { + return err + } + if _, ok := findMatchingRule(path, adapter.ResourceConfig().ProvidedIDFields); !ok { + continue + } + ch.Action = deployplan.UpdateWithID + ch.Reason = deployplan.ReasonParentRenamed + } + return nil +} + // shouldSkipNormalized skips a change that is a false diff caused by UC API // normalization: the API strips trailing slashes from storage URLs // (normalize_slash). The direct engine saves local config to state, so without diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 2d051dc21b9..f50de986918 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -271,6 +271,64 @@ func TestRemoteAlreadySetGuards(t *testing.T) { } } +// TestFollowParentRenames pins when an ID field that references a renamed parent is +// re-keyed (update_id) instead of recreated: only a provided_id_field of a resource with +// DoUpdateWithID, and only if it was under that parent before the rename. +func TestFollowParentRenames(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + tests := []struct { + name string + resource string + field string + old string + expectedAction deployplan.ActionType + }{ + { + name: "child of the renamed parent follows it", + resource: "volumes", + field: "schema_name", + old: "s_old", + expectedAction: deployplan.UpdateWithID, + }, + { + name: "child moved from another parent is recreated", + resource: "volumes", + field: "schema_name", + old: "other", + expectedAction: deployplan.Recreate, + }, + { + name: "resource without DoUpdateWithID is recreated", + resource: "registered_models", + field: "schema_name", + old: "s_old", + expectedAction: deployplan.Recreate, + }, + { + name: "recreate_on_changes field is recreated", + resource: "volumes", + field: "storage_location", + old: "s_old", + expectedAction: deployplan.Recreate, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + adapter, ok := adapters[tt.resource] + require.True(t, ok) + ch := &deployplan.ChangeDesc{Action: deployplan.Recreate, Reason: "id_field", Old: tt.old, New: "s_new", Remote: tt.old} + parentRenames := map[string]*deployplan.ChangeDesc{ + tt.field: {Action: deployplan.UpdateWithID, Reason: "id_changes", Old: "s_old", New: "s_new", Remote: "s_old"}, + } + require.NoError(t, followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames)) + assert.Equal(t, tt.expectedAction, ch.Action) + }) + } +} + // TestRecreateBackendDefault pins the implicit backend-default behavior for // recreate_on_changes fields (recreateOrBackendDefault): an immutable field the // config never set (old and new nil) but the backend populated (remote set) is left diff --git a/bundle/direct/dresources/configs/schemas.yml b/bundle/direct/dresources/configs/schemas.yml index 8ef27e71d15..02f68e4afe9 100644 --- a/bundle/direct/dresources/configs/schemas.yml +++ b/bundle/direct/dresources/configs/schemas.yml @@ -1,9 +1,10 @@ provided_id_fields: - # UC lowercases identifier names; remote returns "myschema" for config "MySchema". - - field: name - reason: id_field - field: catalog_name reason: id_field +updatable_id_fields: + # UC lowercases identifier names; remote returns "myschema" for config "MySchema". + - field: name + reason: id_changes recreate_on_changes: - field: storage_root reason: immutable diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index d80dac584bc..9b7bdd43b51 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -64,20 +64,22 @@ func (r *ResourceSchema) DoCreate(ctx context.Context, config *catalog.CreateSch // migrate invariant (schema.yml is excluded there, and schema_tf_compat.yml covers migrate instead). var schemaForceSend = []string{"Comment", "CustomMaxRetentionHours"} -// DoUpdate updates the schema in place and returns remote state. -func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (*catalog.SchemaInfo, error) { - updateRequest := catalog.UpdateSchema{ +func makeUpdateSchema(id, newName string, config *catalog.CreateSchema) catalog.UpdateSchema { + return catalog.UpdateSchema{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, EnablePredictiveOptimization: "", // Not supported by DABs FullName: id, - NewName: "", // We recreate schemas on name change intentionally. + NewName: newName, Owner: "", // Not supported by DABs Properties: config.Properties, ForceSendFields: utils.FilterFields[catalog.UpdateSchema](append(slices.Clone(schemaForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "NewName", "Owner"), } +} - response, err := r.client.Schemas.Update(ctx, updateRequest) +// DoUpdate updates the schema in place and returns remote state. +func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (*catalog.SchemaInfo, error) { + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(id, "", config)) if err != nil { return nil, err } @@ -89,6 +91,30 @@ func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalo return response, nil } +// DoUpdateWithID renames the schema in place when the name changes; the ID is the full name, so it changes too. +func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (string, *catalog.SchemaInfo, error) { + nameFromID, err := getNameFromID(id) + if err != nil { + return "", nil, err + } + + newName := "" + if config.Name != nameFromID { + newName = config.Name + } + + // The catalog comes from config: a renamed catalog moves the schema with it, leaving id + // stale (see followParentRenames). + fullName := config.CatalogName + "." + nameFromID + + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, config)) + if err != nil || response == nil { + return "", nil, err + } + + return response.FullName, response, nil +} + func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *catalog.CreateSchema) error { return r.client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{ FullName: id, diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 7cd1eaba1ce..168de23f1c1 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -86,7 +86,7 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalo func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ *PlanEntry) (string, *catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, - Name: id, + Name: "", // Initialized below NewName: "", // Initialized below if needed Owner: "", // Not supported by DABs @@ -100,6 +100,10 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * } nameFromID := items[len(items)-1] + // The parents come from config: a renamed catalog or schema moves the volume with it, + // leaving id stale (see followParentRenames). + updateRequest.Name = config.CatalogName + "." + config.SchemaName + "." + nameFromID + if config.Name != nameFromID { updateRequest.NewName = config.Name } diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 9759b4b1a19..325910f93a4 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -110,6 +110,7 @@ func (s *FakeWorkspace) CatalogsUpdate(req Request, name string) Response { // Delete the old entry and create with new name delete(s.Catalogs, name) + s.moveCatalogChildren(name, updateRequest.NewName) name = updateRequest.NewName } @@ -121,3 +122,25 @@ func (s *FakeWorkspace) CatalogsUpdate(req Request, name string) Response { Body: existing, } } + +// moveCatalogChildren mirrors UC, where the schemas and volumes of a renamed catalog move with it. +func (s *FakeWorkspace) moveCatalogChildren(oldCatalogName, newCatalogName string) { + for key, schema := range s.Schemas { + if schema.CatalogName != oldCatalogName { + continue + } + delete(s.Schemas, key) + schema.CatalogName = newCatalogName + schema.FullName = newCatalogName + "." + schema.Name + s.Schemas[schema.FullName] = schema + } + for key, volume := range s.Volumes { + if volume.CatalogName != oldCatalogName { + continue + } + delete(s.Volumes, key) + volume.CatalogName = newCatalogName + volume.FullName = newCatalogName + "." + volume.SchemaName + "." + volume.Name + s.Volumes[volume.FullName] = volume + } +} diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 8094c93acc6..d114ce289f8 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -86,7 +86,7 @@ func (s *FakeWorkspace) SchemasUpdate(req Request, name string) Response { return *errResponse } - var schemaUpdate catalog.SchemaInfo + var schemaUpdate catalog.UpdateSchema if err := json.Unmarshal(req.Body, &schemaUpdate); err != nil { return Response{ @@ -97,12 +97,34 @@ func (s *FakeWorkspace) SchemasUpdate(req Request, name string) Response { applyUpdatedFields(&existing, schemaUpdate, fields) + if schemaUpdate.NewName != "" { + delete(s.Schemas, name) + oldName := existing.Name + // UC normalizes schema names to lowercase. + existing.Name = strings.ToLower(schemaUpdate.NewName) + existing.FullName = existing.CatalogName + "." + existing.Name + s.moveSchemaVolumes(existing.CatalogName, oldName, existing.Name) + } + existing.UpdatedAt = nowMilli() existing.UpdatedBy = s.CurrentUser().UserName - s.Schemas[name] = existing + s.Schemas[existing.FullName] = existing return Response{ Body: existing, } } + +// moveSchemaVolumes mirrors UC, where the volumes of a renamed schema move with it. +func (s *FakeWorkspace) moveSchemaVolumes(catalogName, oldSchemaName, newSchemaName string) { + for key, volume := range s.Volumes { + if volume.CatalogName != catalogName || volume.SchemaName != oldSchemaName { + continue + } + delete(s.Volumes, key) + volume.SchemaName = newSchemaName + volume.FullName = catalogName + "." + newSchemaName + "." + volume.Name + s.Volumes[volume.FullName] = volume + } +} diff --git a/libs/testserver/volumes.go b/libs/testserver/volumes.go index 91f29a94e4e..6cc324557e0 100644 --- a/libs/testserver/volumes.go +++ b/libs/testserver/volumes.go @@ -51,6 +51,16 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { defer s.LockUnlock()() + if _, exists := s.Volumes[volume.FullName]; exists { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Volume '%s' already exists", volume.FullName), + }, + } + } + s.Volumes[volume.FullName] = volume // Register the volume path as a directory so that fs cp works without From 9283a86335adac54e220999af016fb3fba3ac75b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 13:46:05 +0100 Subject: [PATCH 02/21] Link PR in changelog fragment Co-authored-by: Isaac --- .nextchanges/bundles/schema-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md index bc30b0568b4..517c74045d7 100644 --- a/.nextchanges/bundles/schema-rename.md +++ b/.nextchanges/bundles/schema-rename.md @@ -1 +1 @@ -* Rename a schema in place when its `name` changes, instead of deleting and recreating it (direct engine). Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#PR](https://github.com/databricks/cli/pull/PR)) +* Rename a schema in place when its `name` changes, instead of deleting and recreating it (direct engine). Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7343](https://github.com/databricks/cli/pull/7343)) From af435baaa2c0be9f5a78c0cb64307f2a5f4ef980 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 14:00:34 +0100 Subject: [PATCH 03/21] testserver: keep volume create lenient A captured terraform fixture (migrate/command/basic) deploys the same volume name from two targets. The plan goldens already catch a regression to recreate, so the duplicate check is not needed. Co-authored-by: Isaac --- libs/testserver/volumes.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/libs/testserver/volumes.go b/libs/testserver/volumes.go index 6cc324557e0..91f29a94e4e 100644 --- a/libs/testserver/volumes.go +++ b/libs/testserver/volumes.go @@ -51,16 +51,6 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { defer s.LockUnlock()() - if _, exists := s.Volumes[volume.FullName]; exists { - return Response{ - StatusCode: http.StatusBadRequest, - Body: map[string]string{ - "error_code": "RESOURCE_ALREADY_EXISTS", - "message": fmt.Sprintf("Volume '%s' already exists", volume.FullName), - }, - } - } - s.Volumes[volume.FullName] = volume // Register the volume path as a directory so that fs cp works without From 44ce1a63c9a51268d754e812f04641a186f88d97 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 15:46:05 +0100 Subject: [PATCH 04/21] Drop engine mention from changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/schema-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md index 517c74045d7..2ac01106dbe 100644 --- a/.nextchanges/bundles/schema-rename.md +++ b/.nextchanges/bundles/schema-rename.md @@ -1 +1 @@ -* Rename a schema in place when its `name` changes, instead of deleting and recreating it (direct engine). Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7343](https://github.com/databricks/cli/pull/7343)) +* Rename a schema in place when its `name` changes, instead of deleting and recreating it. Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7343](https://github.com/databricks/cli/pull/7343)) From 355a09eedca364c3e8c33a578a6f894cbad9511d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 15:52:56 +0100 Subject: [PATCH 05/21] acc: drop text plan where the JSON plan already shows it Also restores the grants/schemas/change_name golden for the in-place rename, which the rebase onto denik/grants-rekey dropped. Co-authored-by: Isaac --- .../resources/catalogs/rename/output.txt | 52 ++++++++++--------- .../bundle/resources/catalogs/rename/script | 3 +- .../resources/schemas/rename/output.txt | 20 +++---- .../bundle/resources/schemas/rename/script | 3 +- .../schemas/rename_with_volume/output.txt | 36 ++++++------- .../schemas/rename_with_volume/script | 3 +- 6 files changed, 58 insertions(+), 59 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index a4a6e81a101..c0b51984440 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -39,40 +39,42 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ === UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them ->>> [CLI] bundle plan -update catalogs.catalog1 -update schemas.schema1 -update volumes.volume1 - -Plan: 0 to add, 3 to change, 0 to delete, 0 unchanged - >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } } }, "resources.schemas.schema1": { - "catalog_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } } }, "resources.volumes.volume1": { - "catalog_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } } } } diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index d9421c00eb8..aecf603a512 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -21,8 +21,7 @@ title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" -trace $CLI bundle plan -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values(.changes | with_entries(select(.value.action != "skip")))' +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' title "Deploy renames the catalog and updates the schema and the volume at their new paths" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index a213c84394b..b2d3dbeb1e4 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -21,19 +21,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test-schema- test-schema-renamed- === The plan renames the schema in place instead of recreating it ->>> [CLI] bundle plan -update schemas.schema1 - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged - >>> [CLI] bundle plan --output json { - "name": { + "resources.schemas.schema1": { "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + } } } diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index 24dacb067dc..1e5044d9ffc 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -20,8 +20,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "The plan renames the schema in place instead of recreating it" -trace $CLI bundle plan -trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' | normalize_uc_payload.py +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index 17f9535e737..bb4775ffb6d 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -31,30 +31,30 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test-schema- test-schema-renamed- === UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it ->>> [CLI] bundle plan -update schemas.schema1 -update volumes.volume1 - -Plan: 0 to add, 2 to change, 0 to delete, 0 unchanged - >>> [CLI] bundle plan --output json { "resources.schemas.schema1": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } } }, "resources.volumes.volume1": { - "schema_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } } } } diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index 478233c03aa..3197694b3a4 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -21,8 +21,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" -trace $CLI bundle plan -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values(.changes | with_entries(select(.value.action != "skip")))' +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' title "Deploy renames the schema and updates the volume at its new path" trace $CLI bundle deploy From 2ce04ec897f0c1b6d34ccfd768bf3bfc1fb40f9e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 16:00:17 +0100 Subject: [PATCH 06/21] acc: show only the plan field each rename test is about Also restores the grants/schemas/change_name golden for the in-place rename. Co-authored-by: Isaac --- .../resources/catalogs/rename/output.txt | 41 +++---------------- .../bundle/resources/catalogs/rename/script | 2 +- .../grants/schemas/change_name/output.txt | 21 +++------- .../resources/schemas/rename/output.txt | 17 +++----- .../bundle/resources/schemas/rename/script | 2 +- .../schemas/rename_with_volume/output.txt | 29 +++---------- .../schemas/rename_with_volume/script | 2 +- 7 files changed, 24 insertions(+), 90 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index c0b51984440..29b5fe9ef22 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -41,42 +41,11 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them >>> [CLI] bundle plan --output json { - "resources.catalogs.catalog1": { - "action": "update_id", - "changes": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" - } - } - }, - "resources.schemas.schema1": { - "action": "update_id", - "changes": { - "catalog_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" - } - } - }, - "resources.volumes.volume1": { - "action": "update_id", - "changes": { - "catalog_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" - } - } - } + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" } === Deploy renames the catalog and updates the schema and the volume at their new paths diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index aecf603a512..fee34a9d751 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -21,7 +21,7 @@ title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes.catalog_name' title "Deploy renames the catalog and updates the schema and the volume at their new paths" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/grants/schemas/change_name/output.txt b/acceptance/bundle/resources/grants/schemas/change_name/output.txt index 6c4e069ee0c..b7009427f4b 100644 --- a/acceptance/bundle/resources/grants/schemas/change_name/output.txt +++ b/acceptance/bundle/resources/grants/schemas/change_name/output.txt @@ -54,28 +54,19 @@ schemas schema1.grants id='schema/main.test-schema-[UNIQUE_NAME]' === Deploy moves the schema to the new name and applies the grants there >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.schema1 -Recreated schemas.schema1 +Updated schemas.schema1 Updated schemas.schema1.grants Files: 1 uploaded, 0 deleted -Resources: 1 created, 1 changed, 1 deleted, 0 unchanged +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged >>> print_requests.py //unity-catalog { - "method": "DELETE", + "method": "PATCH", "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", - "q": { - "force": "true" - } -} -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/schemas", "body": { - "catalog_name": "main", - "name": "test-schema-renamed-[UNIQUE_NAME]" + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" } } { diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index b2d3dbeb1e4..253928c8d6c 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -23,18 +23,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan renames the schema in place instead of recreating it >>> [CLI] bundle plan --output json { - "resources.schemas.schema1": { - "action": "update_id", - "changes": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" - } - } - } + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" } === Deploy renames the schema without a deletion prompt diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index 1e5044d9ffc..ec7a4694578 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -20,7 +20,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "The plan renames the schema in place instead of recreating it" -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' +trace $CLI bundle plan --output json | jq '.plan["resources.schemas.schema1"].changes.name' title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index bb4775ffb6d..0e943e097c3 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -33,30 +33,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it >>> [CLI] bundle plan --output json { - "resources.schemas.schema1": { - "action": "update_id", - "changes": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" - } - } - }, - "resources.volumes.volume1": { - "action": "update_id", - "changes": { - "schema_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" - } - } - } + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" } === Deploy renames the schema and updates the volume at its new path diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index 3197694b3a4..383f22a7c29 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -21,7 +21,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes.schema_name' title "Deploy renames the schema and updates the volume at its new path" trace $CLI bundle deploy From fbc4bf0f5c128f3bbe9084f6a5bfe8209cd5af53 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:26:00 +0100 Subject: [PATCH 07/21] acc: point migrate recreate comments at schema-catalog-change schema-rename now renames in place, so it no longer covers the schema recreate. Co-authored-by: Isaac --- acceptance/bundle/migrate/auto/schema-storage-change/script | 2 +- acceptance/bundle/migrate/auto/volume-type-change/script | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/migrate/auto/schema-storage-change/script b/acceptance/bundle/migrate/auto/schema-storage-change/script index c0ab15856de..ef97ca99a82 100644 --- a/acceptance/bundle/migrate/auto/schema-storage-change/script +++ b/acceptance/bundle/migrate/auto/schema-storage-change/script @@ -2,7 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= # Testserver-only: an explicit storage_root needs a registered external location on a real # workspace, so this is not Cloud-enabled. The recreate/refusal class it covers is verified -# on cloud by schema-rename and pipeline-storage-change. +# on cloud by schema-catalog-change and pipeline-storage-change. title "Replay the prior terraform deployment (captured from a real terraform run)" trace replay_tfstate.py terraform-requests.json terraform.tfstate diff --git a/acceptance/bundle/migrate/auto/volume-type-change/script b/acceptance/bundle/migrate/auto/volume-type-change/script index 6b02a55f247..ea5ebdb1499 100644 --- a/acceptance/bundle/migrate/auto/volume-type-change/script +++ b/acceptance/bundle/migrate/auto/volume-type-change/script @@ -2,7 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= # Testserver-only: changing volume_type to EXTERNAL needs a storage_location backed by a # registered external location on a real workspace, so this is not Cloud-enabled. The -# recreate class it covers is verified on cloud by schema-rename / registered-model-rename. +# recreate class it covers is verified on cloud by schema-catalog-change / registered-model-rename. # volume_type is both a provided id field and a recreate_on_changes field. reconcileIDFields # records the deployed value, so the migration's plan sees a real change and recreates. Volumes From cf616d2077471e41169df833422a71151096a20e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:35:27 +0100 Subject: [PATCH 08/21] acc: show the whole plan in the rename tests Co-authored-by: Isaac --- .../resources/catalogs/rename/output.txt | 111 +++++++++++++++++- .../bundle/resources/catalogs/rename/script | 3 +- .../resources/schemas/rename/output.txt | 45 ++++++- .../bundle/resources/schemas/rename/script | 2 +- .../schemas/rename_with_volume/output.txt | 83 ++++++++++++- .../schemas/rename_with_volume/script | 3 +- 6 files changed, 229 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 29b5fe9ef22..a0d4de8b2d1 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -41,11 +41,112 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them >>> [CLI] bundle plan --output json { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][2], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "myschema", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", + "volume_id": "[UUID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "storage_location": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" + } + } + } } === Deploy renames the catalog and updates the schema and the volume at their new paths diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index fee34a9d751..4bb99866eef 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -21,7 +21,8 @@ title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" -trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes.catalog_name' +# Drop the remote_state fields that the test server omits or fills differently from UC. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type title "Deploy renames the catalog and updates the schema and the volume at their new paths" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index 253928c8d6c..a2c446f8133 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -23,11 +23,46 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan renames the schema in place instead of recreating it >>> [CLI] bundle plan --output json { - "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "COMMENT1", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "COMMENT1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + } + } } === Deploy renames the schema without a deletion prompt diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index ec7a4694578..3a16724e09a 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -20,7 +20,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "The plan renames the schema in place instead of recreating it" -trace $CLI bundle plan --output json | jq '.plan["resources.schemas.schema1"].changes.name' +trace $CLI bundle plan --output json | jq .plan title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index 0e943e097c3..437e6285edd 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -33,11 +33,84 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it >>> [CLI] bundle plan --output json { - "action": "update_id", - "reason": "parent_renamed", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-[UNIQUE_NAME]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", + "volume_id": "[UUID]", + "volume_type": "MANAGED" + }, + "changes": { + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + }, + "storage_location": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" + } + } + } } === Deploy renames the schema and updates the volume at its new path diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index 383f22a7c29..ab582ad9496 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -21,7 +21,8 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" -trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes.schema_name' +# Drop the remote_state fields that the test server omits or fills differently from UC. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id title "Deploy renames the schema and updates the volume at its new path" trace $CLI bundle deploy From eea8c69c326e0347a1f854bed59b7b7c24e7bc55 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:49:52 +0100 Subject: [PATCH 09/21] direct: don't send an empty update to a child moved by its parent's rename UC moves schemas and volumes with their renamed catalog or schema, so the re-key needs no update. Read the child instead to record its full name as UC spells it. Co-authored-by: Isaac --- .../resources/catalogs/rename/output.txt | 17 +------- .../bundle/resources/catalogs/rename/script | 2 +- .../schemas/rename_with_volume/output.txt | 9 +--- .../schemas/rename_with_volume/script | 2 +- bundle/deployplan/plan.go | 11 +++++ bundle/deployplan/plan_test.go | 41 +++++++++++++++++++ bundle/direct/dresources/schema.go | 12 +++++- bundle/direct/dresources/volume.go | 12 +++++- 8 files changed, 78 insertions(+), 28 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index a0d4de8b2d1..71101a2c013 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -149,7 +149,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== Deploy renames the catalog and updates the schema and the volume at their new paths +=== Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 @@ -168,21 +168,6 @@ Resources: 0 created, 3 changed, 0 deleted, 0 unchanged "new_name": "test_catalog_renamed_[UNIQUE_NAME]" } } -{ - "method": "PATCH", - "path": "/api/2.1/unity-catalog/schemas/test_catalog_renamed_[UNIQUE_NAME].myschema", - "body": { - "comment": "", - "custom_max_retention_hours": 0 - } -} -{ - "method": "PATCH", - "path": "/api/2.1/unity-catalog/volumes/test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", - "body": { - "comment": "" - } -} === The old catalog is gone and the schema and the volume are under the new one old catalog is gone diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index 4bb99866eef..1a20a59c050 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -24,7 +24,7 @@ title "UC moves the schema and the volume with the catalog, so the plan re-keys # Drop the remote_state fields that the test server omits or fills differently from UC. trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type -title "Deploy renames the catalog and updates the schema and the volume at their new paths" +title "Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request" trace $CLI bundle deploy trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index 437e6285edd..d92d0cfbb20 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -113,7 +113,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== Deploy renames the schema and updates the volume at its new path +=== Deploy renames the schema. UC moves the volume with it, so the volume gets no request >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 @@ -131,13 +131,6 @@ Resources: 0 created, 2 changed, 0 deleted, 0 unchanged "new_name": "test-schema-renamed-[UNIQUE_NAME]" } } -{ - "method": "PATCH", - "path": "/api/2.1/unity-catalog/volumes/main.test-schema-renamed-[UNIQUE_NAME].myvolume", - "body": { - "comment": "" - } -} === The old schema is gone and the volume is under the new one >>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index ab582ad9496..3837a1b3b2f 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -24,7 +24,7 @@ title "UC moves the volume with the schema, so the plan re-keys the volume inste # Drop the remote_state fields that the test server omits or fills differently from UC. trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id -title "Deploy renames the schema and updates the volume at its new path" +title "Deploy renames the schema. UC moves the volume with it, so the volume gets no request" trace $CLI bundle deploy trace print_requests.py //unity-catalog diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index d53de5eee48..f8055e51dd1 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -235,6 +235,17 @@ func (c *Changes) HasChangeExcept(prefixes ...string) bool { return false } +// OnlyParentRenamed reports whether every actionable change follows a parent's rename +// (ReasonParentRenamed). The backend has already moved such a resource, so only its ID changes. +func (c *Changes) OnlyParentRenamed() bool { + for _, change := range *c { + if change.Action != Skip && change.Reason != ReasonParentRenamed { + return false + } + } + return true +} + func (p *Plan) GetActions() []Action { actions := make([]Action, 0, len(p.Plan)) for key, entry := range p.Plan { diff --git a/bundle/deployplan/plan_test.go b/bundle/deployplan/plan_test.go index cee00f42c44..dad1020fcfb 100644 --- a/bundle/deployplan/plan_test.go +++ b/bundle/deployplan/plan_test.go @@ -65,3 +65,44 @@ func TestHasChange(t *testing.T) { }) } } + +func TestOnlyParentRenamed(t *testing.T) { + parentRenamed := &deployplan.ChangeDesc{Action: deployplan.UpdateWithID, Reason: deployplan.ReasonParentRenamed} + tests := []struct { + name string + changes deployplan.Changes + want bool + }{ + { + name: "parent renamed", + changes: deployplan.Changes{"schema_name": parentRenamed}, + want: true, + }, + { + name: "skip change is ignored", + changes: deployplan.Changes{ + "schema_name": parentRenamed, + "storage_location": {Action: deployplan.Skip, Reason: "immutable_backend_value"}, + }, + want: true, + }, + { + name: "own field also changes", + changes: deployplan.Changes{ + "schema_name": parentRenamed, + "comment": {Action: deployplan.Update}, + }, + want: false, + }, + { + name: "own rename", + changes: deployplan.Changes{"name": {Action: deployplan.UpdateWithID, Reason: "id_changes"}}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.changes.OnlyParentRenamed()) + }) + } +} diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 9b7bdd43b51..5cc53580265 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -92,7 +92,7 @@ func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalo } // DoUpdateWithID renames the schema in place when the name changes; the ID is the full name, so it changes too. -func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (string, *catalog.SchemaInfo, error) { +func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateSchema, entry *PlanEntry) (string, *catalog.SchemaInfo, error) { nameFromID, err := getNameFromID(id) if err != nil { return "", nil, err @@ -107,6 +107,16 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * // stale (see followParentRenames). fullName := config.CatalogName + "." + nameFromID + if entry.Changes.OnlyParentRenamed() { + // Nothing to update. Read the schema to record its full name as UC spells it: UC + // lowercases names, config may not. + response, err := r.DoRead(ctx, fullName) + if err != nil { + return "", nil, err + } + return response.FullName, response, nil + } + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, config)) if err != nil || response == nil { return "", nil, err diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 168de23f1c1..0f51f668842 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -83,7 +83,7 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalo return response, err } -func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ *PlanEntry) (string, *catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, entry *PlanEntry) (string, *catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: "", // Initialized below @@ -104,6 +104,16 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * // leaving id stale (see followParentRenames). updateRequest.Name = config.CatalogName + "." + config.SchemaName + "." + nameFromID + if entry.Changes.OnlyParentRenamed() { + // Nothing to update. Read the volume to record its full name as UC spells it: UC + // lowercases names, config may not. + response, err := r.DoRead(ctx, updateRequest.Name) + if err != nil { + return "", nil, err + } + return response.FullName, response, nil + } + if config.Name != nameFromID { updateRequest.NewName = config.Name } From ff812aa7cb9992d25e724ebda88a000c36cdef87 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:53:01 +0100 Subject: [PATCH 10/21] acc: update, rename and recreate a volume after its schema is renamed Co-authored-by: Isaac --- .../rename_then_change/databricks.yml.tmpl | 17 ++ .../schemas/rename_then_change/out.test.toml | 2 + .../schemas/rename_then_change/output.txt | 276 ++++++++++++++++++ .../schemas/rename_then_change/script | 50 ++++ .../schemas/rename_then_change/test.toml | 7 + 5 files changed, 352 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/script create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/test.toml diff --git a/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl new file mode 100644 index 00000000000..663acec3be0 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema1-$UNIQUE_NAME + schema2: + catalog_name: main + name: test-schema2-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + comment: c1 diff --git a/acceptance/bundle/resources/schemas/rename_then_change/out.test.toml b/acceptance/bundle/resources/schemas/rename_then_change/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_then_change/output.txt b/acceptance/bundle/resources/schemas/rename_then_change/output.txt new file mode 100644 index 00000000000..625c432a739 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/output.txt @@ -0,0 +1,276 @@ + +=== Deploy two schemas and a volume in the first one +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created schemas.schema2 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema1-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema2-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "c1", + "name": "myvolume", + "schema_name": "test-schema1-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the volume's schema: the volume is re-keyed without a request +>>> update_file.py databricks.yml test-schema1- test-schema1-renamed- + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema1-renamed-[UNIQUE_NAME]" + } +} + +=== After the rename, update, update_id and recreate of the volume all reach it at its new path +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "c2" + } +} + +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "c2", + "new_name": "myvolume2" + } +} + +>>> update_file.py databricks.yml schemas.schema1.name schemas.schema2.name + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +Recreated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume2" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "c2", + "name": "myvolume2", + "schema_name": "test-schema2-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the volume's new schema: the volume is re-keyed again +>>> update_file.py databricks.yml test-schema2- test-schema2-renamed- + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema2 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema2-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema2-renamed-[UNIQUE_NAME]" + } +} + +=== Update, update_id and recreate work again, moving the volume back to the first schema +>>> update_file.py databricks.yml comment: c2 comment: c3 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume2", + "body": { + "comment": "c3" + } +} + +>>> update_file.py databricks.yml name: myvolume2 name: myvolume + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume2", + "body": { + "comment": "c3", + "new_name": "myvolume" + } +} + +>>> update_file.py databricks.yml schemas.schema2.name schemas.schema1.name + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +Recreated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "c3", + "name": "myvolume", + "schema_name": "test-schema1-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the first schema back: the volume is re-keyed to its original path +>>> update_file.py databricks.yml test-schema1-renamed- test-schema1- + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-renamed-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema1-[UNIQUE_NAME]" + } +} + +=== The state tracks the current paths and the plan is a no-op +>>> [CLI] bundle summary --output json +{ + "schemas": { + "schema1": "main.test-schema1-[UNIQUE_NAME]", + "schema2": "main.test-schema2-renamed-[UNIQUE_NAME]" + }, + "volumes": { + "volume1": "main.test-schema1-[UNIQUE_NAME].myvolume" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] volumes read main.test-schema1-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema1-[UNIQUE_NAME].myvolume", + "comment": "c3" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.schemas.schema2 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + delete resources.schemas.schema2 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/schemas/rename_then_change/script b/acceptance/bundle/resources/schemas/rename_then_change/script new file mode 100644 index 00000000000..d89bfb882f8 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/script @@ -0,0 +1,50 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +deploy() { + trace $CLI bundle deploy "$@" + trace print_requests.py //unity-catalog +} + +title "Deploy two schemas and a volume in the first one" +deploy + +title "Rename the volume's schema: the volume is re-keyed without a request" +trace update_file.py databricks.yml test-schema1- test-schema1-renamed- +deploy + +title "After the rename, update, update_id and recreate of the volume all reach it at its new path" +trace update_file.py databricks.yml "comment: c1" "comment: c2" +deploy +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +deploy +trace update_file.py databricks.yml schemas.schema1.name schemas.schema2.name +deploy --auto-approve + +title "Rename the volume's new schema: the volume is re-keyed again" +trace update_file.py databricks.yml test-schema2- test-schema2-renamed- +deploy + +title "Update, update_id and recreate work again, moving the volume back to the first schema" +trace update_file.py databricks.yml "comment: c2" "comment: c3" +deploy +trace update_file.py databricks.yml "name: myvolume2" "name: myvolume" +deploy +trace update_file.py databricks.yml schemas.schema2.name schemas.schema1.name +deploy --auto-approve + +title "Rename the first schema back: the volume is re-keyed to its original path" +trace update_file.py databricks.yml test-schema1-renamed- test-schema1- +deploy + +title "The state tracks the current paths and the plan is a no-op" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' +trace $CLI bundle plan +trace $CLI volumes read "main.test-schema1-${UNIQUE_NAME}.myvolume" | jq '{full_name, comment}' diff --git a/acceptance/bundle/resources/schemas/rename_then_change/test.toml b/acceptance/bundle/resources/schemas/rename_then_change/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From 9b6464dd4e51356b3c50477445d0eaad9d59b96a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:57:50 +0100 Subject: [PATCH 11/21] acc: sort requests of the parallel schema creates Co-authored-by: Isaac --- .../bundle/resources/schemas/rename_then_change/output.txt | 2 +- acceptance/bundle/resources/schemas/rename_then_change/script | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/schemas/rename_then_change/output.txt b/acceptance/bundle/resources/schemas/rename_then_change/output.txt index 625c432a739..ed430978177 100644 --- a/acceptance/bundle/resources/schemas/rename_then_change/output.txt +++ b/acceptance/bundle/resources/schemas/rename_then_change/output.txt @@ -8,7 +8,7 @@ Created volumes.volume1 Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog { "method": "POST", "path": "/api/2.1/unity-catalog/schemas", diff --git a/acceptance/bundle/resources/schemas/rename_then_change/script b/acceptance/bundle/resources/schemas/rename_then_change/script index d89bfb882f8..452af92148c 100644 --- a/acceptance/bundle/resources/schemas/rename_then_change/script +++ b/acceptance/bundle/resources/schemas/rename_then_change/script @@ -14,7 +14,9 @@ deploy() { } title "Deploy two schemas and a volume in the first one" -deploy +trace $CLI bundle deploy +# The two schemas are created in parallel. +trace print_requests.py --sort //unity-catalog title "Rename the volume's schema: the volume is re-keyed without a request" trace update_file.py databricks.yml test-schema1- test-schema1-renamed- From 2b7427850ebce4b4d68ca1fc590bc775416b7bdd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 12:07:31 +0100 Subject: [PATCH 12/21] acc: a job reading a volume's full_name and id follows its schema's rename Co-authored-by: Isaac --- .../databricks.yml.tmpl | 21 +++++ .../rename_with_volume_ref/out.test.toml | 2 + .../schemas/rename_with_volume_ref/output.txt | 87 +++++++++++++++++++ .../schemas/rename_with_volume_ref/script | 30 +++++++ .../schemas/rename_with_volume_ref/test.toml | 7 ++ 5 files changed, 147 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/script create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl new file mode 100644 index 00000000000..2624f14c7cf --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl @@ -0,0 +1,21 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + jobs: + job1: + name: test-job-$UNIQUE_NAME + parameters: + - name: volume_full_name + default: ${resources.volumes.volume1.full_name} + - name: volume_id + default: ${resources.volumes.volume1.id} diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt new file mode 100644 index 00000000000..47697e156fd --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt @@ -0,0 +1,87 @@ + +=== Deploy a schema, a volume in it and a job that reads the volume's full_name and id +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.job1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] jobs get [JOB_ID] +[ + { + "default": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "volume_full_name" + }, + { + "default": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "volume_id" + } +] + +=== Rename the schema +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== The volume's full_name and id change with the schema, so the job is updated too +>>> [CLI] bundle plan +update jobs.job1 +update schemas.schema1 +update volumes.volume1 + +Plan: 0 to add, 3 to change, 0 to delete, 0 unchanged + +=== Deploy passes the volume's new full_name and id to the job +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated jobs.job1 +Updated schemas.schema1 +Updated volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 3 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +>>> [CLI] jobs get [JOB_ID] +[ + { + "default": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "volume_full_name" + }, + { + "default": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "volume_id" + } +] + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.job1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/script b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script new file mode 100644 index 00000000000..0b1627161b3 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script @@ -0,0 +1,30 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema, a volume in it and a job that reads the volume's full_name and id" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +JOB_ID=$($CLI bundle summary --output json | jq -r .resources.jobs.job1.id) +add_repl "$JOB_ID" JOB_ID +trace $CLI jobs get "$JOB_ID" | jq .settings.parameters + +title "Rename the schema" +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "The volume's full_name and id change with the schema, so the job is updated too" +trace $CLI bundle plan + +title "Deploy passes the volume's new full_name and id to the job" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI jobs get "$JOB_ID" | jq .settings.parameters + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml b/acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From 331e7e599b6c72b9ce17c565b9bcc817426745fd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 12:58:54 +0100 Subject: [PATCH 13/21] acc: drop cloud-specific fields from the rename plans Azure and GCP return their own volume storage_location and predictive optimization metastore, and UC may touch a schema's updated_at after create. Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename/output.txt | 6 ------ acceptance/bundle/resources/catalogs/rename/script | 4 ++-- acceptance/bundle/resources/schemas/rename/output.txt | 11 ++--------- acceptance/bundle/resources/schemas/rename/script | 3 ++- .../resources/schemas/rename_with_volume/output.txt | 11 ----------- .../resources/schemas/rename_with_volume/script | 4 ++-- 6 files changed, 8 insertions(+), 31 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 71101a2c013..6534dc8620b 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -128,7 +128,6 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "name": "myvolume", "owner": "[USERNAME]", "schema_name": "myschema", - "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "volume_id": "[UUID]", "volume_type": "MANAGED" }, @@ -139,11 +138,6 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "test_catalog_[UNIQUE_NAME]", "new": "test_catalog_renamed_[UNIQUE_NAME]", "remote": "test_catalog_[UNIQUE_NAME]" - }, - "storage_location": { - "action": "skip", - "reason": "immutable_backend_value", - "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } } } diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index 1a20a59c050..50ee123b666 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -21,8 +21,8 @@ title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" -# Drop the remote_state fields that the test server omits or fills differently from UC. -trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location title "Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index a2c446f8133..d14df16f2e1 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -37,21 +37,14 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "catalog_name": "main", "catalog_type": "MANAGED_CATALOG", "comment": "COMMENT1", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS], "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, "enable_predictive_optimization": "INHERIT", "full_name": "main.test-schema-[UNIQUE_NAME]", "metastore_id": "[UUID]", "name": "test-schema-[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" + "schema_id": "[UUID]" }, "changes": { "name": { diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index 3a16724e09a..bd155193e9e 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -20,7 +20,8 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "The plan renames the schema in place instead of recreating it" -trace $CLI bundle plan --output json | jq .plan +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by effective_predictive_optimization_flag title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index d92d0cfbb20..260614b1f8a 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -47,11 +47,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_at": [UNIX_TIME_MILLIS][0], "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, "enable_predictive_optimization": "INHERIT", "full_name": "main.test-schema-[UNIQUE_NAME]", "name": "test-schema-[UNIQUE_NAME]", @@ -92,7 +87,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "name": "myvolume", "owner": "[USERNAME]", "schema_name": "test-schema-[UNIQUE_NAME]", - "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "volume_id": "[UUID]", "volume_type": "MANAGED" }, @@ -103,11 +97,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]", "remote": "test-schema-[UNIQUE_NAME]" - }, - "storage_location": { - "action": "skip", - "reason": "immutable_backend_value", - "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } } } diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index 3837a1b3b2f..f3591d9f38b 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -21,8 +21,8 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" -# Drop the remote_state fields that the test server omits or fills differently from UC. -trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location title "Deploy renames the schema. UC moves the volume with it, so the volume gets no request" trace $CLI bundle deploy From 88f9ef8d3e8d83e81532fc01957c4bac0f801f4e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 14:09:32 +0100 Subject: [PATCH 14/21] acc: show preserved schema and volume ids through replacements Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename/output.txt | 16 +++++++--------- .../bundle/resources/catalogs/rename/script | 14 +++++--------- .../bundle/resources/schemas/rename/output.txt | 10 ++++------ .../bundle/resources/schemas/rename/script | 13 +++---------- .../schemas/rename_with_volume/output.txt | 10 ++++------ .../resources/schemas/rename_with_volume/script | 13 +++---------- 6 files changed, 26 insertions(+), 50 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 6534dc8620b..07843301bcc 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -88,7 +88,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[SCHEMA_ID]" }, "changes": { "catalog_name": { @@ -128,7 +128,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "name": "myvolume", "owner": "[USERNAME]", "schema_name": "myschema", - "volume_id": "[UUID]", + "volume_id": "[VOLUME_ID]", "volume_type": "MANAGED" }, "changes": { @@ -163,25 +163,23 @@ Resources: 0 created, 3 changed, 0 deleted, 0 unchanged } } -=== The old catalog is gone and the schema and the volume are under the new one +=== The old catalog is gone and the schema and the volume are under the new one, with their original ids old catalog is gone >>> [CLI] schemas get test_catalog_renamed_[UNIQUE_NAME].myschema { "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", - "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]" + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" } >>> [CLI] volumes read test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume { "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", - "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]" + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]" } -=== The schema_id and volume_id are preserved, so these are the same objects -schema_id is unchanged -volume_id is unchanged - === The state tracks the new paths >>> [CLI] bundle summary --output json { diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index 50ee123b666..e84ef070fc8 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -14,8 +14,8 @@ trap cleanup EXIT title "Deploy a catalog with a schema and a volume in it" trace $CLI bundle deploy trace print_requests.py //unity-catalog -SCHEMA_ID=$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id) -VOLUME_ID=$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id) +add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ @@ -28,15 +28,11 @@ title "Deploy renames the catalog. UC moves the schema and the volume with it, s trace $CLI bundle deploy trace print_requests.py //unity-catalog -title "The old catalog is gone and the schema and the volume are under the new one\n" +title "The old catalog is gone and the schema and the volume are under the new one, with their original ids\n" # Not traced: the "not found" message differs between UC and the test server. $CLI catalogs get "$OLD_CATALOG" &> /dev/null && echo "old catalog still exists" || echo "old catalog is gone" -trace $CLI schemas get "$NEW_CATALOG.myschema" | jq '{full_name, catalog_name}' -trace $CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq '{full_name, catalog_name}' - -title "The schema_id and volume_id are preserved, so these are the same objects\n" -[ "$($CLI schemas get "$NEW_CATALOG.myschema" | jq -r .schema_id)" = "$SCHEMA_ID" ] && echo "schema_id is unchanged" || echo "schema_id changed" -[ "$($CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq -r .volume_id)" = "$VOLUME_ID" ] && echo "volume_id is unchanged" || echo "volume_id changed" +trace $CLI schemas get "$NEW_CATALOG.myschema" | jq '{full_name, catalog_name, schema_id}' +trace $CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq '{full_name, catalog_name, volume_id}' title "The state tracks the new paths" trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index d14df16f2e1..275d84698ac 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -44,7 +44,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "metastore_id": "[UUID]", "name": "test-schema-[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[SCHEMA_ID]" }, "changes": { "name": { @@ -76,7 +76,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } } -=== The schema is only reachable under the new name +=== The schema is only reachable under the new name, with its original schema_id >>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. @@ -84,12 +84,10 @@ Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. { "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", "name": "test-schema-renamed-[UNIQUE_NAME]", - "comment": "COMMENT1" + "comment": "COMMENT1", + "schema_id": "[SCHEMA_ID]" } -=== The schema_id is preserved, so this is the same schema -schema_id is unchanged - === The state and summary track the new name >>> [CLI] bundle summary --output json { diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index bd155193e9e..c8617676747 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -14,7 +14,7 @@ trap cleanup EXIT title "Deploy the schema" trace $CLI bundle deploy trace print_requests.py //unity-catalog -SCHEMA_ID=$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id) +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- @@ -27,16 +27,9 @@ title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy trace print_requests.py //unity-catalog -title "The schema is only reachable under the new name" +title "The schema is only reachable under the new name, with its original schema_id" trace musterr $CLI schemas get "$OLD_SCHEMA" -trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, name, comment}' - -title "The schema_id is preserved, so this is the same schema\n" -if [ "$($CLI schemas get "$NEW_SCHEMA" | jq -r .schema_id)" = "$SCHEMA_ID" ]; then - echo "schema_id is unchanged" -else - echo "schema_id changed" -fi +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, name, comment, schema_id}' title "The state and summary track the new name" trace $CLI bundle summary --output json | jq '.resources.schemas.schema1 | {id, name}' diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index 260614b1f8a..37458a8f7bf 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -87,7 +87,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "name": "myvolume", "owner": "[USERNAME]", "schema_name": "test-schema-[UNIQUE_NAME]", - "volume_id": "[UUID]", + "volume_id": "[VOLUME_ID]", "volume_type": "MANAGED" }, "changes": { @@ -121,19 +121,17 @@ Resources: 0 created, 2 changed, 0 deleted, 0 unchanged } } -=== The old schema is gone and the volume is under the new one +=== The old schema is gone and the volume is under the new one, with its original volume_id >>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. >>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume { "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", - "schema_name": "test-schema-renamed-[UNIQUE_NAME]" + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]" } -=== The volume_id is preserved, so this is the same volume -volume_id is unchanged - === The state tracks the new path >>> [CLI] bundle summary --output json "main.test-schema-renamed-[UNIQUE_NAME].myvolume" diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index f3591d9f38b..fd5830bf18a 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -15,7 +15,7 @@ trap cleanup EXIT title "Deploy a schema with a volume in it" trace $CLI bundle deploy trace print_requests.py //unity-catalog -VOLUME_ID=$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id) +add_repl "$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id)" VOLUME_ID title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- @@ -28,16 +28,9 @@ title "Deploy renames the schema. UC moves the volume with it, so the volume get trace $CLI bundle deploy trace print_requests.py //unity-catalog -title "The old schema is gone and the volume is under the new one" +title "The old schema is gone and the volume is under the new one, with its original volume_id" trace musterr $CLI schemas get "$OLD_SCHEMA" -trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, schema_name}' - -title "The volume_id is preserved, so this is the same volume\n" -if [ "$($CLI volumes read "$NEW_VOLUME" | jq -r .volume_id)" = "$VOLUME_ID" ]; then - echo "volume_id is unchanged" -else - echo "volume_id changed" -fi +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, schema_name, volume_id}' title "The state tracks the new path" trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' From 27ceb824744e3f0e7783695a6df69124fd870473 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 14:26:20 +0100 Subject: [PATCH 15/21] direct: build the re-keyed child's ID from config instead of reading it Like the other DoUpdateWithID implementations, take the new ID from config, so a child moved by its parent's rename needs no request at all. Co-authored-by: Isaac --- bundle/direct/dresources/schema.go | 9 ++------- bundle/direct/dresources/volume.go | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 5cc53580265..8a928cb8989 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -108,13 +108,8 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * fullName := config.CatalogName + "." + nameFromID if entry.Changes.OnlyParentRenamed() { - // Nothing to update. Read the schema to record its full name as UC spells it: UC - // lowercases names, config may not. - response, err := r.DoRead(ctx, fullName) - if err != nil { - return "", nil, err - } - return response.FullName, response, nil + // UC already moved the schema with its catalog; only the ID changes. + return fullName, nil, nil } response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, config)) diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 0f51f668842..445cd826314 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -105,13 +105,8 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * updateRequest.Name = config.CatalogName + "." + config.SchemaName + "." + nameFromID if entry.Changes.OnlyParentRenamed() { - // Nothing to update. Read the volume to record its full name as UC spells it: UC - // lowercases names, config may not. - response, err := r.DoRead(ctx, updateRequest.Name) - if err != nil { - return "", nil, err - } - return response.FullName, response, nil + // UC already moved the volume with its parent; only the ID changes. + return updateRequest.Name, nil, nil } if config.Name != nameFromID { From fb2400b5fcb51def3446000f15265808b9631de7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 16:26:08 +0100 Subject: [PATCH 16/21] direct: follow a parent rename that an interrupted deploy already applied A deploy can stop after renaming a schema but before re-keying its volume. The next plan then saw no rename in progress, read the volume at its old path, got 404, and planned a create, which fails on UC with RESOURCE_ALREADY_EXISTS. An update_id now records the old values of the fields it changes (renamed_from) in the same state write as the new ID. A child whose parent is not being renamed in the plan follows that recorded rename instead, and is not planned as a create just because its read at the old ID found nothing. Co-authored-by: Isaac --- .../rename_interrupted/databricks.yml.tmpl | 14 ++ .../schemas/rename_interrupted/out.test.toml | 2 + .../schemas/rename_interrupted/output.txt | 157 ++++++++++++++++++ .../schemas/rename_interrupted/script | 38 +++++ .../schemas/rename_interrupted/test.toml | 29 ++++ .../permission_level_migration/output.txt | 5 +- bundle/direct/apply.go | 28 +++- bundle/direct/bundle_plan.go | 41 +++-- bundle/direct/bundle_plan_test.go | 4 +- bundle/direct/dstate/dms.go | 12 +- bundle/direct/dstate/dms_test.go | 13 ++ bundle/direct/dstate/state.go | 25 ++- bundle/direct/dstate/state_test.go | 17 ++ 13 files changed, 355 insertions(+), 30 deletions(-) create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/script create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/test.toml diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl new file mode 100644 index 00000000000..2964856d08e --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + comment: c1 diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml b/acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt new file mode 100644 index 00000000000..091a96903cb --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -0,0 +1,157 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change the volume's comment. The CLI is killed on the volume's update, after the schema's rename is saved +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> errcode [CLI] bundle deploy -q +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +[PROCESS_KILLED] + +Exit code: [KILLED] + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== The next plan still re-keys the volume, whose state has the old schema +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "custom_max_retention_hours": 0, + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][1], + "updated_by": "[USERNAME]" + }, + "changes": { + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "c2", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "comment": { + "action": "update", + "old": "c1", + "new": "c2" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } + } +} + +=== It can, because the schema's old name was saved together with its rename +>>> print_state.py +{ + "__id__": "main.test-schema-renamed-[UNIQUE_NAME]", + "renamed_from": { + "name": "test-schema-[UNIQUE_NAME]" + } +} + +=== The next deploy updates the volume at its new path +>>> [CLI] bundle deploy --force-lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "c2" + } +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "comment": "c2", + "volume_id": "[VOLUME_ID]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve --force-lock +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/script b/acceptance/bundle/resources/schemas/rename_interrupted/script new file mode 100644 index 00000000000..ca0d499bb7e --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/script @@ -0,0 +1,38 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +NEW_VOLUME="main.test-schema-renamed-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve --force-lock + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "main.test-schema-${UNIQUE_NAME}.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema and change the volume's comment. The CLI is killed on the volume's update, after the schema's rename is saved" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml "comment: c1" "comment: c2" +kill_after.py "PATCH /api/2.1/unity-catalog/volumes/$NEW_VOLUME" 0 1 +# -q: the per-resource lines race the kill signal. +trace errcode $CLI bundle deploy -q +trace print_requests.py //unity-catalog + +title "The next plan still re-keys the volume, whose state has the old schema" +trace $CLI bundle plan --output json | jq .plan + +title "It can, because the schema's old name was saved together with its rename" +trace print_state.py | jq '.state["resources.schemas.schema1"] | {__id__, renamed_from}' + +title "The next deploy updates the volume at its new path" +trace $CLI bundle deploy --force-lock +trace print_requests.py //unity-catalog +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, comment, volume_id}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/test.toml b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml new file mode 100644 index 00000000000..284671b7e15 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml @@ -0,0 +1,29 @@ +# Local-only: the test server kills the CLI mid-deploy (kill_after.py), which a real workspace cannot. +Cloud = false + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# KillCaller needs the CLI's pid. A kill leaves no state in DMS to recover from, so DMS is not run. +Env.DATABRICKS_CLI_TEST_PID = "1" +EnvMatrix.DMS = [""] + +# Normalize how a killed process shows up on each OS, as in bundle/deploy/wal. +[[Repls]] +Old = 'script: line \d+:\s+\d+ Killed(: 9)?\s+"\$@"' +New = '[PROCESS_KILLED]' + +[[Repls]] +Old = '(\n>>> errcode [^\n]+\n)\nExit code:' +New = "${1}[PROCESS_KILLED]\n\nExit code:" + +[[Repls]] +Old = 'Exit code: 137' +New = 'Exit code: [KILLED]' + +[[Repls]] +Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' +New = '${1}[KILLED]' diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index 2091843525f..a217b0067b9 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -54,7 +54,10 @@ Resources: 1 created, 1 changed, 0 deleted, 0 unchanged "node": "resources.jobs.my_job", "label": "${resources.jobs.my_job.id}" } - ] + ], + "renamed_from": { + "object_id": "/jobs/123" + } } } } diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 9cdd5a4f505..718b13d1ba4 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -85,7 +85,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, nil) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -183,7 +183,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn, nil) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -228,7 +228,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, renamedFrom(planEntry.Changes)) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -321,7 +321,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn, nil) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -333,7 +333,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // with content hashes, see dresources.CompactState) before persisting it. Fields already // smaller than a hash placeholder are persisted as is. Sensitive fields are replaced by a // placeholder value so secrets are never written to disk in plaintext. -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, renamedFrom map[string]any) error { compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), state) if err != nil { return fmt.Errorf("compacting state: %w", err) @@ -341,7 +341,23 @@ func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentSta if err := zeroSensitiveFields(d.Adapter, compacted); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveState(ctx, d.ResourceKey, newID, compacted, dependsOn) + return db.SaveStateRenamed(ctx, d.ResourceKey, newID, compacted, dependsOn, renamedFrom) +} + +// renamedFrom returns the old values of the fields an update_id changes, for the state entry's +// RenamedFrom. +func renamedFrom(changes deployplan.Changes) map[string]any { + var result map[string]any + for path, change := range changes { + if change.Action != deployplan.UpdateWithID { + continue + } + if result == nil { + result = make(map[string]any) + } + result[path] = change.Old + } + return result } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 6042aa4f33c..4bfce2f5536 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -377,15 +377,16 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - err = followParentRenames(adapter, entry.Changes, parentRenames) + followedRename, err := followParentRenames(adapter, entry.Changes, parentRenames) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) return false } - if remoteState == nil { + if remoteState == nil && !followedRename { // Even if local action is "recreate" which is higher than "create", we should still pick "create" here - // because we know remote does not exist. + // because we know remote does not exist. A child that followed its parent's rename is not + // missing: it moved, and the read used its old ID. action = deployplan.Create } else { action = getMaxAction(entry.Changes) @@ -711,6 +712,9 @@ func (b *DeploymentBundle) parentRenames(resourceKey string) (map[string]*deploy } // renameChange returns the change to field of resourceKey if it is part of an update_id, nil otherwise. +// +// When this plan does not rename the field, the rename recorded with the resource's state counts +// too: a deploy interrupted after renaming the parent leaves its children under the old name. func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan.ChangeDesc, error) { entry, err := b.Plan.ReadLockEntry(resourceKey) if err != nil { @@ -719,14 +723,25 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. defer b.Plan.ReadUnlockEntry(resourceKey) // A missing entry is reported by resolveReferences. - if entry == nil || entry.Action != deployplan.UpdateWithID { + if entry == nil { + return nil, nil + } + if change := entry.Changes[field]; change != nil && change.Action == deployplan.UpdateWithID { + return change, nil + } + // A parent that is created or recreated is a new object; UC does not move children into it. + if !entry.Action.KeepsID() && entry.Action != deployplan.UpdateWithID { + return nil, nil + } + dbentry, ok := b.StateDB.GetResourceEntry(resourceKey) + if !ok { return nil, nil } - change := entry.Changes[field] - if change == nil || change.Action != deployplan.UpdateWithID { + old, ok := dbentry.RenamedFrom[field] + if !ok { return nil, nil } - return change, nil + return &deployplan.ChangeDesc{Action: deployplan.UpdateWithID, Old: old}, nil } // followParentRenames plans an update_id instead of a recreate for an ID field that changes @@ -736,10 +751,13 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. // // The old values must match too: a child moved to the renamed parent from somewhere else // still has to be recreated. -func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) error { +// +// It reports whether any change follows a rename. +func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) (bool, error) { if !adapter.HasDoUpdateWithID() { - return nil + return false, nil } + followed := false for pathString, parentChange := range parentRenames { ch := changes[pathString] if ch == nil || ch.Action != deployplan.Recreate || !structdiff.IsEqual(ch.Old, parentChange.Old) { @@ -747,15 +765,16 @@ func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes } path, err := structpath.ParsePath(pathString) if err != nil { - return err + return false, err } if _, ok := findMatchingRule(path, adapter.ResourceConfig().ProvidedIDFields); !ok { continue } ch.Action = deployplan.UpdateWithID ch.Reason = deployplan.ReasonParentRenamed + followed = true } - return nil + return followed, nil } // shouldSkipNormalized skips a change that is a false diff caused by UC API diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index f50de986918..7d735755080 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -323,8 +323,10 @@ func TestFollowParentRenames(t *testing.T) { parentRenames := map[string]*deployplan.ChangeDesc{ tt.field: {Action: deployplan.UpdateWithID, Reason: "id_changes", Old: "s_old", New: "s_new", Remote: "s_old"}, } - require.NoError(t, followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames)) + followed, err := followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames) + require.NoError(t, err) assert.Equal(t, tt.expectedAction, ch.Action) + assert.Equal(t, tt.expectedAction == deployplan.UpdateWithID, followed) }) } } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 57e10b8d4b4..3f38ddc2bd8 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -12,8 +12,9 @@ import ( // config so depends_on survives the round trip: DMS has no field for dependency edges, and // nesting them in the config would collide with resource fields of the same name. type RecordedState struct { - State json.RawMessage `json:"state"` - DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + RenamedFrom map[string]any `json:"renamed_from,omitempty"` } // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS @@ -54,8 +55,9 @@ func stateEntry(res dms.Resource) (ResourceEntry, error) { } return ResourceEntry{ - ID: res.ID, - State: recorded.State, - DependsOn: recorded.DependsOn, + ID: res.ID, + State: recorded.State, + DependsOn: recorded.DependsOn, + RenamedFrom: recorded.RenamedFrom, }, nil } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index ab4e1bc017f..18a1aceddcc 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -44,6 +44,19 @@ func TestApplyDMSState(t *testing.T) { "resources.pipelines.bar": {ID: "456"}, }, }, + { + name: "renamed_from comes back from the envelope", + recorded: []dms.Resource{ + {Key: "resources.schemas.foo", ID: "main.new", State: `{"state":{"name":"new"},"renamed_from":{"name":"old"}}`}, + }, + want: map[string]ResourceEntry{ + "resources.schemas.foo": { + ID: "main.new", + State: json.RawMessage(`{"name":"new"}`), + RenamedFrom: map[string]any{"name": "old"}, + }, + }, + }, { // Nil is what a list holding nothing returns, and it means a successful deploy of // nothing rather than missing data. diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 384246913f4..2745e2bf057 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -168,6 +168,12 @@ type ResourceEntry struct { ID string `json:"__id__"` State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + + // RenamedFrom holds the old values of the fields an in-place ID change (update_id) changed, + // keyed by field path, until the resource is saved again. It is saved in the same write as + // the new ID, so a child that an interrupted deploy left under the old name can still follow + // the rename. + RenamedFrom map[string]any `json:"renamed_from,omitempty"` } type WALEntry struct { @@ -274,7 +280,7 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e var recorded json.RawMessage if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { var err error - recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn, RenamedFrom: entry.RenamedFrom}) if err != nil { // Nothing the caller can act on, so record the failure without the state. recorded = nil @@ -300,6 +306,12 @@ func NewDatabase(lineage string, serial int) Database { // SaveState records the resource's state after an operation was applied to it. func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { + return db.SaveStateRenamed(ctx, key, newID, state, dependsOn, nil) +} + +// SaveStateRenamed is SaveState for an ID changed in place: renamedFrom (see ResourceEntry) is +// recorded together with the new ID. +func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, renamedFrom map[string]any) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -313,9 +325,10 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta return err } entry := ResourceEntry{ - ID: newID, - State: json.RawMessage(jsonMessage), - DependsOn: dependsOn, + ID: newID, + State: json.RawMessage(jsonMessage), + DependsOn: dependsOn, + RenamedFrom: renamedFrom, } switch db.mode { @@ -329,7 +342,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, RenamedFrom: renamedFrom}) if err != nil { return err } @@ -344,7 +357,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta } db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, RenamedFrom: renamedFrom}) if err != nil { return err } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index c5bfea2783b..9e22d15b8a1 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -38,6 +38,23 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { mustFinalize(t, &db2) } +func TestSaveStateRenamedRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) + require.NoError(t, db.SaveStateRenamed(t.Context(), "schemas.foo", "main.new", map[string]string{"name": "new"}, nil, map[string]any{"name": "old"})) + mustFinalize(t, &db) + + var db2 DeploymentState + require.NoError(t, db2.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + entry, ok := db2.GetResourceEntry("schemas.foo") + require.True(t, ok) + assert.Equal(t, "main.new", entry.ID) + assert.Equal(t, map[string]any{"name": "old"}, entry.RenamedFrom) + mustFinalize(t, &db2) +} + func TestFinalizeWithoutResourceWrites(t *testing.T) { for _, mode := range []Mode{WritingFile, WritingDeploymentHistoryShadow} { path := filepath.Join(t.TempDir(), "state.json") From e0225427453a600b61fae187b7fed55a9048377b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 12:10:50 +0100 Subject: [PATCH 17/21] direct: find a moved schema or volume by its stable ID instead of renamed_from When the read at the recorded ID finds nothing, the plan reads at the ID the config gives the resource and takes what it finds only if its stable ID (schema_id, volume_id, now saved in the state entry and the DMS envelope) matches. This covers a deploy that stopped after renaming the parent and a rename done out of band, and never adopts a different object of the same name. It replaces renamed_from, which missed repeated renames and could not tell a rename from a recreate. The test server now deletes children with a forced catalog/schema delete and rejects a catalog create under a taken name, like UC. The migrate fixtures dropped the out-of-band catalog create that the scripts already run. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 5 +- .../bundle/dms/declined-deploy/output.txt | 5 +- .../dms/deployment-metadata-change/output.txt | 2 +- acceptance/bundle/dms/empty-grants/output.txt | 5 +- .../bundle/dms/failed-recreate/output.txt | 5 +- .../operation-upload-fails-once/output.txt | 8 +- .../bundle/dms/successful-recreate/output.txt | 10 +- .../terraform-requests.json | 7 - .../terraform-requests.json | 7 - .../schemas/rename_interrupted/output.txt | 124 ++++++++---------- .../schemas/rename_interrupted/script | 8 +- .../permission_level_migration/output.txt | 5 +- bundle/deployplan/plan.go | 2 + bundle/direct/apply.go | 35 +++-- bundle/direct/bundle_plan.go | 94 +++++++++---- bundle/direct/bundle_plan_test.go | 4 +- bundle/direct/dresources/README.md | 4 + bundle/direct/dresources/adapter.go | 60 +++++++++ bundle/direct/dresources/schema.go | 10 ++ bundle/direct/dresources/volume.go | 10 ++ bundle/direct/dstate/dms.go | 14 +- bundle/direct/dstate/dms_test.go | 10 +- bundle/direct/dstate/state.go | 30 ++--- bundle/direct/dstate/state_test.go | 8 +- libs/testserver/catalogs.go | 33 +++++ libs/testserver/handlers.go | 4 +- libs/testserver/schemas.go | 19 +++ 27 files changed, 334 insertions(+), 194 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 9badc1e4d5e..e7f69ccda56 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -146,6 +146,7 @@ def get_resources(target): "id": resource.get("resource_id"), "state": envelope.get("state") or {}, "depends_on": envelope.get("depends_on") or [], + "stable_id": envelope.get("stable_id"), } page_token = listed.get("next_page_token") if not page_token: @@ -154,12 +155,14 @@ def get_resources(target): def get_recorded_state(target): """The recorded resources in the on-disk state file's `state` shape (resources. -> - {__id__, state, depends_on}), so a recording run prints the same shape as a non-recording one.""" + {__id__, state, depends_on, stable_id}), so a recording run prints the same shape as a non-recording one.""" state = {} for key, value in sorted(get_resources(target).items()): entry = {"__id__": value["id"], "state": value["state"]} if value["depends_on"]: entry["depends_on"] = value["depends_on"] + if value["stable_id"]: + entry["stable_id"] = value["stable_id"] state[f"resources.{key}"] = entry return state diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 02b3d5450f9..5b6771f3d7b 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -99,7 +99,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" - } + }, + "stable_id": "[UUID]" } } } diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 94f0a68a122..8e96186b9ff 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_metadata_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index c12e022ced2..ceeacc47b2b 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -52,7 +52,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_empty_grants_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -171,7 +171,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "name": "dms_empty_grants_[UNIQUE_NAME]" - } + }, + "stable_id": "[UUID]" } } } diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 2e599afbde7..8be24682946 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -101,7 +101,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "catalog_name": "main", "comment": "v1", "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" - } + }, + "stable_id": "[UUID]" } } } diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 042e9d0669e..8708782f930 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -48,7 +48,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "catalog_name": "main", "comment": "v1", "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" - } + }, + "stable_id": "[UUID]" } } } @@ -95,7 +96,8 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "catalog_name": "dms_other_[UNIQUE_NAME]", "comment": "v1", "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" - } + }, + "stable_id": "[UUID]" } } } @@ -141,7 +143,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "error_message": "", "resource_id": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 41c9d29e8e2..dfbc8d82e8a 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -101,7 +101,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "catalog_name": "main", "comment": "v1", "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" - } + }, + "stable_id": "[UUID]" } } } @@ -159,7 +160,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "error_message": "", "resource_id": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", "sequence_id": 1, - "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -213,7 +214,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "catalog_name": "dms_other_[UNIQUE_NAME]", "comment": "v1", "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" - } + }, + "stable_id": "[UUID]" } } } diff --git a/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json b/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json index 6f4620d81f5..6c27b1fc729 100644 --- a/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json +++ b/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json @@ -1,11 +1,4 @@ [ - { - "method": "POST", - "path": "/api/2.1/unity-catalog/catalogs", - "body": { - "name": "othercat_[UNIQUE_NAME]" - } - }, { "method": "POST", "path": "/api/2.1/unity-catalog/models", diff --git a/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json b/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json index eda99b84d42..e7072606fd3 100644 --- a/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json +++ b/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json @@ -1,11 +1,4 @@ [ - { - "method": "POST", - "path": "/api/2.1/unity-catalog/catalogs", - "body": { - "name": "othercat_[UNIQUE_NAME]" - } - }, { "method": "POST", "path": "/api/2.1/unity-catalog/schemas", diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt index 091a96903cb..b2a40eebbfe 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -29,88 +29,68 @@ Exit code: [KILLED] } } -=== The next plan still re-keys the volume, whose state has the old schema +=== The volume is gone from its recorded path, so the plan looks for it at its path in config +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "stable_id": "[VOLUME_ID]" +} + >>> [CLI] bundle plan --output json { - "resources.schemas.schema1": { - "action": "skip", - "remote_state": { - "browse_only": false, + "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "custom_max_retention_hours": 0, - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "test-schema-renamed-[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][1], - "updated_by": "[USERNAME]" - }, - "changes": { - "comment": { - "action": "skip", - "reason": "empty", - "remote": "" - }, - "custom_max_retention_hours": { - "action": "skip", - "reason": "empty", - "remote": 0 - } + "comment": "c2", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" } }, - "resources.volumes.volume1": { - "depends_on": [ - { - "node": "resources.schemas.schema1", - "label": "${resources.schemas.schema1.name}" - } - ], - "action": "update_id", - "new_state": { - "value": { - "catalog_name": "main", - "comment": "c2", - "name": "myvolume", - "schema_name": "test-schema-renamed-[UNIQUE_NAME]", - "volume_type": "MANAGED" - } + "remote_state": { + "catalog_name": "main", + "comment": "c1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "comment": { + "action": "update", + "old": "c1", + "new": "c2", + "remote": "c1" }, - "changes": { - "comment": { - "action": "update", - "old": "c1", - "new": "c2" - }, - "schema_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]" - } + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "storage_location": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]" } } } -=== It can, because the schema's old name was saved together with its rename ->>> print_state.py -{ - "__id__": "main.test-schema-renamed-[UNIQUE_NAME]", - "renamed_from": { - "name": "test-schema-[UNIQUE_NAME]" - } -} - === The next deploy updates the volume at its new path >>> [CLI] bundle deploy --force-lock Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/script b/acceptance/bundle/resources/schemas/rename_interrupted/script index ca0d499bb7e..196ee5d5faa 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/script +++ b/acceptance/bundle/resources/schemas/rename_interrupted/script @@ -23,11 +23,9 @@ kill_after.py "PATCH /api/2.1/unity-catalog/volumes/$NEW_VOLUME" 0 1 trace errcode $CLI bundle deploy -q trace print_requests.py //unity-catalog -title "The next plan still re-keys the volume, whose state has the old schema" -trace $CLI bundle plan --output json | jq .plan - -title "It can, because the schema's old name was saved together with its rename" -trace print_state.py | jq '.state["resources.schemas.schema1"] | {__id__, renamed_from}' +title "The volume is gone from its recorded path, so the plan looks for it at its path in config" +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, stable_id}' +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' title "The next deploy updates the volume at its new path" trace $CLI bundle deploy --force-lock diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index a217b0067b9..2091843525f 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -54,10 +54,7 @@ Resources: 1 created, 1 changed, 0 deleted, 0 unchanged "node": "resources.jobs.my_job", "label": "${resources.jobs.my_job.id}" } - ], - "renamed_from": { - "object_id": "/jobs/123" - } + ] } } } diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index f8055e51dd1..99d70f7b80b 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -119,6 +119,8 @@ func LoadPlanFromFile(path string) (*Plan, error) { } type PlanEntry struct { + // ID is set when the resource was found at another ID than the one in state: a rename + // moved it there (see findMoved). Applying the entry addresses the resource at ID. ID string `json:"id,omitempty"` DependsOn []DependsOnEntry `json:"depends_on,omitempty"` Action ActionType `json:"action,omitempty"` diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 718b13d1ba4..84274ab56fc 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -44,6 +44,10 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, if oldID == "" { return errors.New("state entry not found") } + if planEntry.ID != "" { + // The plan found the resource at the ID its config gives it: a rename moved it. + oldID = planEntry.ID + } switch actionType { case deployplan.Recreate: @@ -85,7 +89,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, nil) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, d.Adapter.StableID(d.RemoteState)) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -183,7 +187,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn, nil) + err = d.saveState(ctx, db, id, newState, d.DependsOn, d.stableID(planEntry.RemoteState)) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -228,7 +232,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, renamedFrom(planEntry.Changes)) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, d.stableID(planEntry.RemoteState)) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -321,7 +325,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn, nil) + err = d.saveState(ctx, db, id, newState, d.DependsOn, d.stableID(entry.RemoteState)) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -333,7 +337,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // with content hashes, see dresources.CompactState) before persisting it. Fields already // smaller than a hash placeholder are persisted as is. Sensitive fields are replaced by a // placeholder value so secrets are never written to disk in plaintext. -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, renamedFrom map[string]any) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, stableID string) error { compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), state) if err != nil { return fmt.Errorf("compacting state: %w", err) @@ -341,23 +345,16 @@ func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentSta if err := zeroSensitiveFields(d.Adapter, compacted); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveStateRenamed(ctx, d.ResourceKey, newID, compacted, dependsOn, renamedFrom) + return db.SaveStateWithStableID(ctx, d.ResourceKey, newID, compacted, dependsOn, stableID) } -// renamedFrom returns the old values of the fields an update_id changes, for the state entry's -// RenamedFrom. -func renamedFrom(changes deployplan.Changes) map[string]any { - var result map[string]any - for path, change := range changes { - if change.Action != deployplan.UpdateWithID { - continue - } - if result == nil { - result = make(map[string]any) - } - result[path] = change.Old +// stableID returns the stable ID (see dstate.ResourceEntry) of the object being saved: from the +// remote state the operation returned, else from planRemote, the same object as the plan read it. +func (d *DeploymentUnit) stableID(planRemote any) string { + if id := d.Adapter.StableID(d.RemoteState); id != "" { + return id } - return result + return d.Adapter.StableID(planRemote) } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 4bfce2f5536..b35c4e32441 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -334,6 +334,13 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } } + if remoteState == nil { + entry.ID, remoteState, err = findMoved(ctx, adapter, sv, dbentry) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + } // We have a choice whether to include remoteState or remoteStateComparable from below. // Including remoteState because in the near future remoteState is expected to become a superset struct of remoteStateComparable @@ -377,20 +384,23 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - followedRename, err := followParentRenames(adapter, entry.Changes, parentRenames) + err = followParentRenames(adapter, entry.Changes, parentRenames) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) return false } - if remoteState == nil && !followedRename { + if remoteState == nil { // Even if local action is "recreate" which is higher than "create", we should still pick "create" here - // because we know remote does not exist. A child that followed its parent's rename is not - // missing: it moved, and the read used its old ID. + // because we know remote does not exist. action = deployplan.Create } else { action = getMaxAction(entry.Changes) } + if entry.ID != "" { + // Found at a new ID: the state has to follow it there. + action = deployplan.GetHigherAction(action, deployplan.UpdateWithID) + } // Note, this unconditionally stores remoteState. However, it may updated post-deploy, so whether // it can be used for variable resolution depends on several factors, see canReadRemoteCache in LookupReferencePreDeploy @@ -712,9 +722,6 @@ func (b *DeploymentBundle) parentRenames(resourceKey string) (map[string]*deploy } // renameChange returns the change to field of resourceKey if it is part of an update_id, nil otherwise. -// -// When this plan does not rename the field, the rename recorded with the resource's state counts -// too: a deploy interrupted after renaming the parent leaves its children under the old name. func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan.ChangeDesc, error) { entry, err := b.Plan.ReadLockEntry(resourceKey) if err != nil { @@ -723,25 +730,60 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. defer b.Plan.ReadUnlockEntry(resourceKey) // A missing entry is reported by resolveReferences. - if entry == nil { + if entry == nil || entry.Action != deployplan.UpdateWithID { return nil, nil } - if change := entry.Changes[field]; change != nil && change.Action == deployplan.UpdateWithID { - return change, nil - } - // A parent that is created or recreated is a new object; UC does not move children into it. - if !entry.Action.KeepsID() && entry.Action != deployplan.UpdateWithID { + change := entry.Changes[field] + if change == nil || change.Action != deployplan.UpdateWithID { return nil, nil } - dbentry, ok := b.StateDB.GetResourceEntry(resourceKey) - if !ok { - return nil, nil + return change, nil +} + +// findMoved looks for a resource missing from its recorded ID at the ID its config gives it, where a +// rename can have moved it: the resource or its parent renamed out of band, or by a deploy that +// stopped before re-keying it. What is found there is taken only if its stable ID matches the +// recorded one, so an unrelated object with the same name is never adopted; the create then fails +// on it instead. +func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.StructVar, dbentry dstate.ResourceEntry) (string, any, error) { + if !adapter.HasStableID() || !adapter.HasDoUpdateWithID() || dbentry.StableID == "" || hasUnresolvedIDField(adapter, sv) { + return "", nil, nil + } + id, err := adapter.IDFromState(sv.Value) + if err != nil || id == dbentry.ID { + return "", nil, err + } + remote, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { + return adapter.DoRead(ctx, id) + }) + if apierr.IsMissing(err) { + return "", nil, nil } - old, ok := dbentry.RenamedFrom[field] - if !ok { - return nil, nil + if err != nil { + return "", nil, fmt.Errorf("reading id=%q: %w", id, err) } - return &deployplan.ChangeDesc{Action: deployplan.UpdateWithID, Old: old}, nil + if adapter.StableID(remote) != dbentry.StableID { + return "", nil, nil + } + return id, remote, nil +} + +// hasUnresolvedIDField reports whether a field the ID is built from is still an unresolved reference. +func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) bool { + cfg := adapter.ResourceConfig() + for pathString := range sv.Refs { + path, err := structpath.ParsePath(pathString) + if err != nil { + return true + } + if _, ok := findMatchingRule(path, cfg.ProvidedIDFields); ok { + return true + } + if _, ok := findMatchingRule(path, cfg.UpdatableIDFields); ok { + return true + } + } + return false } // followParentRenames plans an update_id instead of a recreate for an ID field that changes @@ -751,13 +793,10 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. // // The old values must match too: a child moved to the renamed parent from somewhere else // still has to be recreated. -// -// It reports whether any change follows a rename. -func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) (bool, error) { +func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) error { if !adapter.HasDoUpdateWithID() { - return false, nil + return nil } - followed := false for pathString, parentChange := range parentRenames { ch := changes[pathString] if ch == nil || ch.Action != deployplan.Recreate || !structdiff.IsEqual(ch.Old, parentChange.Old) { @@ -765,16 +804,15 @@ func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes } path, err := structpath.ParsePath(pathString) if err != nil { - return false, err + return err } if _, ok := findMatchingRule(path, adapter.ResourceConfig().ProvidedIDFields); !ok { continue } ch.Action = deployplan.UpdateWithID ch.Reason = deployplan.ReasonParentRenamed - followed = true } - return followed, nil + return nil } // shouldSkipNormalized skips a change that is a false diff caused by UC API diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 7d735755080..f50de986918 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -323,10 +323,8 @@ func TestFollowParentRenames(t *testing.T) { parentRenames := map[string]*deployplan.ChangeDesc{ tt.field: {Action: deployplan.UpdateWithID, Reason: "id_changes", Old: "s_old", New: "s_new", Remote: "s_old"}, } - followed, err := followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames) - require.NoError(t, err) + require.NoError(t, followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames)) assert.Equal(t, tt.expectedAction, ch.Action) - assert.Equal(t, tt.expectedAction == deployplan.UpdateWithID, followed) }) } } diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index 78952537943..98f10f85228 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -47,6 +47,10 @@ For resources whose create or update is asynchronous (the resource is not immedi If the API may return a slice's elements in a different order between calls (e.g., `depends_on` in job tasks, `privileges` in grants), implement `KeyedSlices` to compare elements by a natural key rather than by index. Without this, every deploy after any reordering shows phantom diffs. +## Moved resources: IDFromState / StableID + +If the ID is built from the resource's own fields (a UC full name) and the backend has an ID that a rename keeps (a volume's `volume_id`), implement `IDFromState` and `StableID` together, along with `DoUpdateWithID`. When nothing is at the recorded ID, the plan reads at the ID the config gives the resource and takes what it finds only if its stable ID matches the one saved in state. This covers a rename done out of band, or by a deploy that stopped before re-keying the resource. + ## Empty states: IsEmptyState If a desired state describes no resource at all (e.g. an empty grants list), implement `IsEmptyState`. The planner omits such a node instead of planning a create, and apply drops its state entry instead of persisting one. diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 80ece24b2be..75c91cdc4d5 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -118,6 +118,17 @@ type IResource interface { // (returned by GET, not 404) that rejects a second DELETE. // Example: func (*ResourceApp) IsGone(remote *AppRemote) bool IsGone(remoteState any) bool + + // [Optional] IDFromState returns the ID the state gives the resource, for resource types whose ID + // is built from their own fields (a UC full name). Implemented together with StableID. + // Example: func (*ResourceVolume) IDFromState(state *catalog.CreateVolumeRequestContent) string + IDFromState(state any) string + + // [Optional] StableID returns the backend's ID for the object that a rename keeps (a volume's + // volume_id). When the resource is not at its recorded ID, the plan looks for it at the ID + // IDFromState gives it, and takes what it finds only if StableID matches the recorded one. + // Example: func (*ResourceVolume) StableID(remote *catalog.VolumeInfo) string + StableID(remoteState any) string } // Adapter wraps resource implementation, validates signatures and type consistency across methods @@ -141,6 +152,8 @@ type Adapter struct { overrideChangeDesc *calladapt.BoundCaller doResize *calladapt.BoundCaller isGone *calladapt.BoundCaller + idFromState *calladapt.BoundCaller + stableID *calladapt.BoundCaller resourceConfig *ResourceLifecycleConfig generatedResourceConfig *ResourceLifecycleConfig @@ -182,6 +195,8 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC waitAfterDelete: nil, overrideChangeDesc: nil, isGone: nil, + idFromState: nil, + stableID: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), keyedSlices: nil, @@ -308,6 +323,16 @@ func (a *Adapter) initMethods(resource any) error { return err } + a.idFromState, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "IDFromState") + if err != nil { + return err + } + + a.stableID, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "StableID") + if err != nil { + return err + } + keyedSlicesCall, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "KeyedSlices") if err != nil { return err @@ -402,6 +427,14 @@ func (a *Adapter) validate() error { validations = append(validations, "IsGone remoteState", a.isGone.InTypes[0], remoteType) } + if (a.idFromState == nil) != (a.stableID == nil) { + return errors.New("IDFromState and StableID must be implemented together") + } + if a.idFromState != nil { + validations = append(validations, "IDFromState state", a.idFromState.InTypes[0], stateType) + validations = append(validations, "StableID remoteState", a.stableID.InTypes[0], remoteType) + } + if a.doUpdateWithID != nil { validations = append(validations, "DoUpdateWithID newState", a.doUpdateWithID.InTypes[2], stateType) // DoUpdateWithID must return (string, remoteType, error) @@ -727,6 +760,33 @@ func (a *Adapter) IsGone(remoteState any) bool { return outs[0].(bool) } +// HasStableID reports whether the resource implements IDFromState and StableID. +func (a *Adapter) HasStableID() bool { + return a.stableID != nil +} + +// IDFromState returns the ID the state gives the resource; see IResource.IDFromState. +func (a *Adapter) IDFromState(state any) (string, error) { + outs, err := a.idFromState.Call(state) + if err != nil { + return "", err + } + return outs[0].(string), nil +} + +// StableID returns the rename-proof ID of the object in remoteState; see IResource.StableID. +// It is empty for a resource type without one and for a nil remote state. +func (a *Adapter) StableID(remoteState any) string { + if a.stableID == nil || remoteState == nil || reflect.ValueOf(remoteState).IsNil() { + return "" + } + outs, err := a.stableID.Call(remoteState) + if err != nil { + return "" + } + return outs[0].(string) +} + // prepareCallRequired prepares a call and ensures the method is found. func prepareCallRequired(resource any, methodName string) (*calladapt.BoundCaller, error) { caller, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), methodName) diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 8a928cb8989..6e1856aa421 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -120,6 +120,16 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * return response.FullName, response, nil } +// IDFromState returns the full name, which is the ID. +func (*ResourceSchema) IDFromState(state *catalog.CreateSchema) string { + return state.CatalogName + "." + state.Name +} + +// StableID returns the schema_id, which a rename keeps. +func (*ResourceSchema) StableID(remote *catalog.SchemaInfo) string { + return remote.SchemaId +} + func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *catalog.CreateSchema) error { return r.client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{ FullName: id, diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 445cd826314..06674d45f96 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -121,6 +121,16 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * return response.FullName, response, nil } +// IDFromState returns the full name, which is the ID. +func (*ResourceVolume) IDFromState(state *catalog.CreateVolumeRequestContent) string { + return state.CatalogName + "." + state.SchemaName + "." + state.Name +} + +// StableID returns the volume_id, which a rename keeps. +func (*ResourceVolume) StableID(remote *catalog.VolumeInfo) string { + return remote.VolumeId +} + func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *catalog.CreateVolumeRequestContent) error { return r.client.Volumes.DeleteByName(ctx, id) } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 3f38ddc2bd8..4842b8de10b 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -12,9 +12,9 @@ import ( // config so depends_on survives the round trip: DMS has no field for dependency edges, and // nesting them in the config would collide with resource fields of the same name. type RecordedState struct { - State json.RawMessage `json:"state"` - DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` - RenamedFrom map[string]any `json:"renamed_from,omitempty"` + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + StableID string `json:"stable_id,omitempty"` } // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS @@ -55,9 +55,9 @@ func stateEntry(res dms.Resource) (ResourceEntry, error) { } return ResourceEntry{ - ID: res.ID, - State: recorded.State, - DependsOn: recorded.DependsOn, - RenamedFrom: recorded.RenamedFrom, + ID: res.ID, + State: recorded.State, + DependsOn: recorded.DependsOn, + StableID: recorded.StableID, }, nil } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 18a1aceddcc..0dc86caf879 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -45,15 +45,15 @@ func TestApplyDMSState(t *testing.T) { }, }, { - name: "renamed_from comes back from the envelope", + name: "stable_id comes back from the envelope", recorded: []dms.Resource{ - {Key: "resources.schemas.foo", ID: "main.new", State: `{"state":{"name":"new"},"renamed_from":{"name":"old"}}`}, + {Key: "resources.schemas.foo", ID: "main.foo", State: `{"state":{"name":"foo"},"stable_id":"uuid-1"}`}, }, want: map[string]ResourceEntry{ "resources.schemas.foo": { - ID: "main.new", - State: json.RawMessage(`{"name":"new"}`), - RenamedFrom: map[string]any{"name": "old"}, + ID: "main.foo", + State: json.RawMessage(`{"name":"foo"}`), + StableID: "uuid-1", }, }, }, diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 2745e2bf057..ecb0cb75ed6 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -169,11 +169,10 @@ type ResourceEntry struct { State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` - // RenamedFrom holds the old values of the fields an in-place ID change (update_id) changed, - // keyed by field path, until the resource is saved again. It is saved in the same write as - // the new ID, so a child that an interrupted deploy left under the old name can still follow - // the rename. - RenamedFrom map[string]any `json:"renamed_from,omitempty"` + // StableID is the backend's ID for the object that survives a rename (a volume's volume_id), + // for resource types that have one. When the object is not at ID any more, the plan looks for + // it at the ID its config gives it and takes what it finds there only if StableID matches. + StableID string `json:"stable_id,omitempty"` } type WALEntry struct { @@ -280,7 +279,7 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e var recorded json.RawMessage if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { var err error - recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn, RenamedFrom: entry.RenamedFrom}) + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn, StableID: entry.StableID}) if err != nil { // Nothing the caller can act on, so record the failure without the state. recorded = nil @@ -306,12 +305,11 @@ func NewDatabase(lineage string, serial int) Database { // SaveState records the resource's state after an operation was applied to it. func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { - return db.SaveStateRenamed(ctx, key, newID, state, dependsOn, nil) + return db.SaveStateWithStableID(ctx, key, newID, state, dependsOn, "") } -// SaveStateRenamed is SaveState for an ID changed in place: renamedFrom (see ResourceEntry) is -// recorded together with the new ID. -func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, renamedFrom map[string]any) error { +// SaveStateWithStableID is SaveState that also records the object's stable ID (see ResourceEntry). +func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, stableID string) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -325,10 +323,10 @@ func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID stri return err } entry := ResourceEntry{ - ID: newID, - State: json.RawMessage(jsonMessage), - DependsOn: dependsOn, - RenamedFrom: renamedFrom, + ID: newID, + State: json.RawMessage(jsonMessage), + DependsOn: dependsOn, + StableID: stableID, } switch db.mode { @@ -342,7 +340,7 @@ func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID stri db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, RenamedFrom: renamedFrom}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, StableID: stableID}) if err != nil { return err } @@ -357,7 +355,7 @@ func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID stri } db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, RenamedFrom: renamedFrom}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, StableID: stableID}) if err != nil { return err } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 9e22d15b8a1..6330a5d99e4 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -38,20 +38,20 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { mustFinalize(t, &db2) } -func TestSaveStateRenamedRoundTrip(t *testing.T) { +func TestSaveStateWithStableIDRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - require.NoError(t, db.SaveStateRenamed(t.Context(), "schemas.foo", "main.new", map[string]string{"name": "new"}, nil, map[string]any{"name": "old"})) + require.NoError(t, db.SaveStateWithStableID(t.Context(), "schemas.foo", "main.foo", map[string]string{"name": "foo"}, nil, "uuid-1")) mustFinalize(t, &db) var db2 DeploymentState require.NoError(t, db2.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) entry, ok := db2.GetResourceEntry("schemas.foo") require.True(t, ok) - assert.Equal(t, "main.new", entry.ID) - assert.Equal(t, map[string]any{"name": "old"}, entry.RenamedFrom) + assert.Equal(t, "main.foo", entry.ID) + assert.Equal(t, "uuid-1", entry.StableID) mustFinalize(t, &db2) } diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 325910f93a4..ff62b2e6959 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -37,6 +37,16 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { } } + if _, exists := s.Catalogs[createRequest.Name]; exists { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "CATALOG_ALREADY_EXISTS", + "message": fmt.Sprintf("Catalog '%s' already exists", createRequest.Name), + }, + } + } + // Echo back every field create accepts: a dropped one makes the next plan see a // phantom change. catalogInfo := catalog.CatalogInfo{ @@ -144,3 +154,26 @@ func (s *FakeWorkspace) moveCatalogChildren(oldCatalogName, newCatalogName strin s.Volumes[volume.FullName] = volume } } + +// CatalogsDelete mirrors UC, where deleting a catalog with force deletes its schemas and volumes too. +func (s *FakeWorkspace) CatalogsDelete(req Request, name string) Response { + defer s.LockUnlock()() + + if _, ok := s.Catalogs[name]; !ok { + return Response{StatusCode: http.StatusNotFound} + } + delete(s.Catalogs, name) + if req.URL.Query().Get("force") == "true" { + for key, schema := range s.Schemas { + if schema.CatalogName == name { + delete(s.Schemas, key) + } + } + for key, volume := range s.Volumes { + if volume.CatalogName == name { + delete(s.Volumes, key) + } + } + } + return Response{} +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e5f4c37a16b..1fa448f4743 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -798,7 +798,7 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("DELETE", "/api/2.1/unity-catalog/schemas/{full_name}", func(req Request) any { - return MapDelete(req.Workspace, req.Workspace.Schemas, req.Vars["full_name"]) + return req.Workspace.SchemasDelete(req, req.Vars["full_name"]) }) // Grants: @@ -826,7 +826,7 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("DELETE", "/api/2.1/unity-catalog/catalogs/{name}", func(req Request) any { - return MapDelete(req.Workspace, req.Workspace.Catalogs, req.Vars["name"]) + return req.Workspace.CatalogsDelete(req, req.Vars["name"]) }) // External Locations: diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index d114ce289f8..8b3403a0204 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -128,3 +128,22 @@ func (s *FakeWorkspace) moveSchemaVolumes(catalogName, oldSchemaName, newSchemaN s.Volumes[volume.FullName] = volume } } + +// SchemasDelete mirrors UC, where deleting a schema with force deletes the volumes in it too. +func (s *FakeWorkspace) SchemasDelete(req Request, fullName string) Response { + defer s.LockUnlock()() + + schema, ok := s.Schemas[fullName] + if !ok { + return Response{StatusCode: http.StatusNotFound} + } + delete(s.Schemas, fullName) + if req.URL.Query().Get("force") == "true" { + for key, volume := range s.Volumes { + if volume.CatalogName == schema.CatalogName && volume.SchemaName == schema.Name { + delete(s.Volumes, key) + } + } + } + return Response{} +} From 3d322f0d2a442916ea185309ac5da9e21a7d658a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 12:11:04 +0100 Subject: [PATCH 18/21] acc: out-of-band renames and recreates of volumes, schemas and catalogs Co-authored-by: Isaac --- .../databricks.yml.tmpl | 16 ++ .../rename_all_out_of_band/out.test.toml | 2 + .../rename_all_out_of_band/output.txt | 174 ++++++++++++++++++ .../catalogs/rename_all_out_of_band/script | 44 +++++ .../catalogs/rename_all_out_of_band/test.toml | 7 + .../rename_out_of_band/databricks.yml.tmpl | 16 ++ .../catalogs/rename_out_of_band/out.test.toml | 2 + .../catalogs/rename_out_of_band/output.txt | 139 ++++++++++++++ .../catalogs/rename_out_of_band/script | 40 ++++ .../catalogs/rename_out_of_band/test.toml | 7 + .../recreate_interrupted/databricks.yml.tmpl | 14 ++ .../recreate_interrupted/out.test.toml | 2 + .../schemas/recreate_interrupted/output.txt | 121 ++++++++++++ .../schemas/recreate_interrupted/script | 36 ++++ .../schemas/recreate_interrupted/test.toml | 30 +++ .../recreate_out_of_band/databricks.yml.tmpl | 13 ++ .../recreate_out_of_band/out.test.toml | 2 + .../schemas/recreate_out_of_band/output.txt | 91 +++++++++ .../schemas/recreate_out_of_band/script | 38 ++++ .../schemas/recreate_out_of_band/test.toml | 7 + .../rename_out_of_band/databricks.yml.tmpl | 13 ++ .../schemas/rename_out_of_band/out.test.toml | 2 + .../schemas/rename_out_of_band/output.txt | 152 +++++++++++++++ .../schemas/rename_out_of_band/script | 44 +++++ .../schemas/rename_out_of_band/test.toml | 7 + .../recreate_out_of_band/databricks.yml.tmpl | 13 ++ .../recreate_out_of_band/out.test.toml | 2 + .../volumes/recreate_out_of_band/output.txt | 81 ++++++++ .../volumes/recreate_out_of_band/script | 34 ++++ .../volumes/recreate_out_of_band/test.toml | 7 + .../rename_out_of_band/databricks.yml.tmpl | 13 ++ .../volumes/rename_out_of_band/out.test.toml | 2 + .../volumes/rename_out_of_band/output.txt | 121 ++++++++++++ .../volumes/rename_out_of_band/script | 41 +++++ .../volumes/rename_out_of_band/test.toml | 7 + 35 files changed, 1340 insertions(+) create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/script create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/script create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/output.txt create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/script create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/test.toml create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/script create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/script create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/script create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/script create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/test.toml diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt new file mode 100644 index 00000000000..7b1f43ba6ce --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -0,0 +1,174 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the volume, the schema and the catalog out of band +>>> [CLI] volumes update test_catalog_[UNIQUE_NAME].myschema.myvolume --new-name myvolume2 +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume2" +} + +>>> [CLI] schemas update test_catalog_[UNIQUE_NAME].myschema --new-name myschema2 +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2" +} + +>>> [CLI] catalogs update test_catalog_[UNIQUE_NAME] --new-name test_catalog_renamed_[UNIQUE_NAME] +{ + "name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +=== Rename all three in the config to match +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "create", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema2", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2", + "name": "myschema2", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myschema", + "new": "myschema2", + "remote": "myschema2" + } + } + }, + "resources.volumes.volume1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume2", + "schema_name": "myschema2", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2", + "name": "myvolume2", + "owner": "[USERNAME]", + "schema_name": "myschema2", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myvolume", + "new": "myvolume2", + "remote": "myvolume2" + }, + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myschema", + "new": "myschema2", + "remote": "myschema2" + } + } + } +} + +=== The catalog's create fails because the renamed catalog has its name +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs +HTTP Status: 400 Bad Request +API error_code: CATALOG_ALREADY_EXISTS +API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists + +Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 + +Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 + +Files: 3 uploaded, 0 deleted + +>>> errcode [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script new file mode 100644 index 00000000000..9645fc0dc02 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -0,0 +1,44 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_CATALOG="test_catalog_${UNIQUE_NAME}" +NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + # The renamed catalog is not where the state looks for it, so destroy cannot remove it. + trace errcode $CLI catalogs delete "$NEW_CATALOG" --force + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location +} + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the volume, the schema and the catalog out of band" +trace $CLI volumes update "$OLD_CATALOG.myschema.myvolume" --new-name myvolume2 | jq "{full_name}" +trace $CLI schemas update "$OLD_CATALOG.myschema" --new-name myschema2 | jq "{full_name}" +trace $CLI catalogs update "$OLD_CATALOG" --new-name "$NEW_CATALOG" | jq "{name}" +print_requests.py //unity-catalog &> LOG.rename_requests + +title "Rename all three in the config to match" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +NEW_SCHEMA="$NEW_CATALOG.myschema2" +NEW_VOLUME="$NEW_SCHEMA.myvolume2" + +title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +plan + +title "The catalog's create fails because the renamed catalog has its name" +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt new file mode 100644 index 00000000000..09bf7199065 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -0,0 +1,139 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the catalog out of band. UC moves the schema and the volume with it +>>> [CLI] catalogs update test_catalog_[UNIQUE_NAME] --new-name test_catalog_renamed_[UNIQUE_NAME] +{ + "name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +=== Rename the catalog in the config to match +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "create", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "myschema", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + } +} + +=== The catalog's create fails because the renamed catalog has its name +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs +HTTP Status: 400 Bad Request +API error_code: CATALOG_ALREADY_EXISTS +API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists + +Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 + +Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 + +Files: 3 uploaded, 0 deleted + +>>> errcode [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script new file mode 100644 index 00000000000..8b161d76a37 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -0,0 +1,40 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_CATALOG="test_catalog_${UNIQUE_NAME}" +NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + # The renamed catalog is not where the state looks for it, so destroy cannot remove it. + trace errcode $CLI catalogs delete "$NEW_CATALOG" --force + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location +} + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the catalog out of band. UC moves the schema and the volume with it" +trace $CLI catalogs update "$OLD_CATALOG" --new-name "$NEW_CATALOG" | jq "{name}" +print_requests.py //unity-catalog &> LOG.rename_requests + +title "Rename the catalog in the config to match" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +NEW_SCHEMA="$NEW_CATALOG.myschema" +NEW_VOLUME="$NEW_SCHEMA.myvolume" + +title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +plan + +title "The catalog's create fails because the renamed catalog has its name" +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl new file mode 100644 index 00000000000..a965908256c --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + storage_root: s3://mybucket/a + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml b/acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt new file mode 100644 index 00000000000..69ff54088bd --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt @@ -0,0 +1,121 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change its storage_root, which recreates it. The CLI is killed on the volume's recreate, after the schema's +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml s3://mybucket/a s3://mybucket/b + +>>> errcode [CLI] bundle deploy -q --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.schema1 + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +[PROCESS_KILLED] + +Exit code: [KILLED] + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "q": { + "force": "true" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "storage_root": "s3://mybucket/b" + } +} + +=== The volume went with the old schema. Nothing with its volume_id is at its new path, so the plan creates it +>>> [CLI] bundle plan --output json +{ + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } +} + +=== The next deploy creates a new volume +>>> [CLI] bundle deploy --force-lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created volumes.volume1 +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[UUID]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve --force-lock +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/script b/acceptance/bundle/resources/schemas/recreate_interrupted/script new file mode 100644 index 00000000000..df501ef6124 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/script @@ -0,0 +1,36 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_VOLUME="main.test-schema-${UNIQUE_NAME}.myvolume" +NEW_VOLUME="main.test-schema-renamed-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve --force-lock + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema and change its storage_root, which recreates it. The CLI is killed on the volume's recreate, after the schema's" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml s3://mybucket/a s3://mybucket/b +kill_after.py "DELETE /api/2.1/unity-catalog/volumes/$OLD_VOLUME" 0 1 +# -q: the per-resource lines race the kill signal. +trace errcode $CLI bundle deploy -q --auto-approve +trace print_requests.py //unity-catalog + +title "The volume went with the old schema. Nothing with its volume_id is at its new path, so the plan creates it" +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' + +title "The next deploy creates a new volume" +trace $CLI bundle deploy --force-lock +trace print_requests.py //unity-catalog +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, volume_id}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml new file mode 100644 index 00000000000..23d6ce8f7fa --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml @@ -0,0 +1,30 @@ +# Local-only: the test server kills the CLI mid-deploy (kill_after.py), which a real workspace +# cannot, and an explicit storage_root needs an external location on a real workspace. +Cloud = false + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# KillCaller needs the CLI's pid. A kill leaves no state in DMS to recover from, so DMS is not run. +Env.DATABRICKS_CLI_TEST_PID = "1" +EnvMatrix.DMS = [""] + +# Normalize how a killed process shows up on each OS, as in bundle/deploy/wal. +[[Repls]] +Old = 'script: line \d+:\s+\d+ Killed(: 9)?\s+"\$@"' +New = '[PROCESS_KILLED]' + +[[Repls]] +Old = '(\n>>> errcode [^\n]+\n)\nExit code:' +New = "${1}[PROCESS_KILLED]\n\nExit code:" + +[[Repls]] +Old = 'Exit code: 137' +New = 'Exit code: [KILLED]' + +[[Repls]] +Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' +New = '${1}[KILLED]' diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml b/acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt new file mode 100644 index 00000000000..f2520fb4368 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt @@ -0,0 +1,91 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Out of band, drop the schema and create another one under a new name, with a volume of the same name +>>> [CLI] schemas delete main.test-schema-[UNIQUE_NAME] --force + +=== With the config renamed to match, the plan finds a schema and a volume at the new names, but their ids differ, so it plans creates +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> print_state.py +{ + "resources.schemas.schema1": { + "__id__": "main.test-schema-[UNIQUE_NAME]", + "stable_id": "[SCHEMA_ID]" + }, + "resources.volumes.volume1": { + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "stable_id": "[VOLUME_ID]" + } +} + +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[NEW_SCHEMA_ID]" +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[NEW_VOLUME_ID]" +} + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } + } +} + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> [CLI] schemas delete main.test-schema-renamed-[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/script b/acceptance/bundle/resources/schemas/recreate_out_of_band/script new file mode 100644 index 00000000000..2d0997ea594 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/script @@ -0,0 +1,38 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + # The schema created out of band is not in the state, so destroy does not remove it. + trace $CLI schemas delete "$NEW_SCHEMA" --force + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Out of band, drop the schema and create another one under a new name, with a volume of the same name" +trace $CLI schemas delete "$OLD_SCHEMA" --force +add_repl "$($CLI schemas create "test-schema-renamed-${UNIQUE_NAME}" main | jq -r .schema_id)" NEW_SCHEMA_ID +add_repl "$($CLI volumes create main "test-schema-renamed-${UNIQUE_NAME}" myvolume MANAGED | jq -r .volume_id)" NEW_VOLUME_ID +print_requests.py //unity-catalog &> LOG.recreate_requests + +title "With the config renamed to match, the plan finds a schema and a volume at the new names, but their ids differ, so it plans creates" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace print_state.py | jq '.state | map_values({__id__, stable_id})' +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' +trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' +plan diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml b/acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt new file mode 100644 index 00000000000..0b951a5e912 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt @@ -0,0 +1,152 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema out of band. UC moves the volume with it +>>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +=== With the config unchanged, the plan creates the schema and the volume again under their configured names +>>> [CLI] bundle plan +create schemas.schema1 +create volumes.volume1 + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +=== With the config renamed to match, the plan finds both at their new names, with the same schema_id and volume_id +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "id": "main.test-schema-renamed-[UNIQUE_NAME]", + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + } + } + } +} + +=== Deploy only records the new names in the state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated volumes.volume1 +Files: 3 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] bundle summary --output json +{ + "schemas": { + "schema1": "main.test-schema-renamed-[UNIQUE_NAME]" + }, + "volumes": { + "volume1": "main.test-schema-renamed-[UNIQUE_NAME].myvolume" + } +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/script b/acceptance/bundle/resources/schemas/rename_out_of_band/script new file mode 100644 index 00000000000..48f43a5ebc4 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/script @@ -0,0 +1,44 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema out of band. UC moves the volume with it" +trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name, schema_id}' +print_requests.py //unity-catalog &> LOG.rename_requests + +title "With the config unchanged, the plan creates the schema and the volume again under their configured names" +trace $CLI bundle plan + +title "With the config renamed to match, the plan finds both at their new names, with the same schema_id and volume_id" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +plan + +title "Deploy only records the new names in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' +trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml b/acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt new file mode 100644 index 00000000000..8ebe0b53337 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt @@ -0,0 +1,81 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Out of band, delete the volume and create another one under a new name +>>> [CLI] volumes delete main.test-schema-[UNIQUE_NAME].myvolume + +=== With the config renamed to match, the plan finds a volume at the new name, but its volume_id differs, so it plans a create +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "stable_id": "[VOLUME_ID]" +} + +>>> [CLI] volumes read main.test-schema-[UNIQUE_NAME].myvolume2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "volume_id": "[NEW_VOLUME_ID]" +} + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume2", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myvolume", + "new": "myvolume2" + } + } + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/script b/acceptance/bundle/resources/volumes/recreate_out_of_band/script new file mode 100644 index 00000000000..ce1a0dacf83 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/script @@ -0,0 +1,34 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +SCHEMA="main.test-schema-${UNIQUE_NAME}" + +cleanup() { + # Deleting the schema deletes the volumes in it, the one created out of band too. + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Out of band, delete the volume and create another one under a new name" +trace $CLI volumes delete "$SCHEMA.myvolume" +NEW_VOLUME_ID=$($CLI volumes create main "test-schema-${UNIQUE_NAME}" myvolume2 MANAGED | jq -r .volume_id) +add_repl "$NEW_VOLUME_ID" NEW_VOLUME_ID +print_requests.py //unity-catalog &> LOG.recreate_requests + +title "With the config renamed to match, the plan finds a volume at the new name, but its volume_id differs, so it plans a create" +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, stable_id}' +trace $CLI volumes read "$SCHEMA.myvolume2" | jq '{full_name, volume_id}' +plan diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml b/acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt new file mode 100644 index 00000000000..80687ae93cc --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt @@ -0,0 +1,121 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the volume out of band +>>> [CLI] volumes update main.test-schema-[UNIQUE_NAME].myvolume --new-name myvolume2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "volume_id": "[VOLUME_ID]" +} + +=== With the config unchanged, the plan creates the volume again under its configured name +>>> [CLI] bundle plan +create volumes.volume1 + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged + +=== With the config renamed to match, the plan finds the volume at its new name, with the same volume_id +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + } + }, + "resources.volumes.volume1": { + "id": "main.test-schema-[UNIQUE_NAME].myvolume2", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume2", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "name": "myvolume2", + "owner": "[USERNAME]", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myvolume", + "new": "myvolume2", + "remote": "myvolume2" + } + } + } +} + +=== Deploy only records the new name in the state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] volumes read main.test-schema-[UNIQUE_NAME].myvolume2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] bundle summary --output json +"main.test-schema-[UNIQUE_NAME].myvolume2" + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/script b/acceptance/bundle/resources/volumes/rename_out_of_band/script new file mode 100644 index 00000000000..c16fde22dbc --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/script @@ -0,0 +1,41 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +SCHEMA="main.test-schema-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the volume out of band" +trace $CLI volumes update "$SCHEMA.myvolume" --new-name myvolume2 | jq '{full_name, volume_id}' +print_requests.py //unity-catalog &> LOG.rename_requests + +title "With the config unchanged, the plan creates the volume again under its configured name" +trace $CLI bundle plan + +title "With the config renamed to match, the plan finds the volume at its new name, with the same volume_id" +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +plan + +title "Deploy only records the new name in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI volumes read "$SCHEMA.myvolume2" | jq '{full_name, volume_id}' +trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml b/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From fc532ceb62375b365a644491b5bc4277ba142a54 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 12:55:50 +0100 Subject: [PATCH 19/21] acc: record stable_id in the user_agent DMS request golden Co-authored-by: Isaac --- acceptance/bundle/user_agent/simple/out.requests.deploy.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.json index 31bc7b0724a..569fd9a0a65 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.json @@ -250,7 +250,8 @@ "state": { "catalog_name": "mycatalog", "name": "myschema" - } + }, + "stable_id": "[UUID]" } } } From a4e026e047cca986e2a4817d1e33e20314d06f71 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 13:25:19 +0100 Subject: [PATCH 20/21] acc: normalize a killed CLI on Windows in the interrupted rename tests Co-authored-by: Isaac --- .../bundle/resources/schemas/recreate_interrupted/test.toml | 6 ++++++ .../bundle/resources/schemas/rename_interrupted/test.toml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml index 23d6ce8f7fa..e149a3f7289 100644 --- a/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml @@ -28,3 +28,9 @@ New = 'Exit code: [KILLED]' [[Repls]] Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' New = '${1}[KILLED]' + +# On Windows a killed CLI leaves no bash "Killed" line. The kill lands right after the +# approval text, so its last line is followed directly by "Exit code: 1". +[[Repls]] +Old = '( recreate resources\.volumes\.volume1)\n\nExit code: 1' +New = "${1}\n[PROCESS_KILLED]\n\nExit code: [KILLED]" diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/test.toml b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml index 284671b7e15..443e541f555 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/test.toml +++ b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml @@ -27,3 +27,9 @@ New = 'Exit code: [KILLED]' [[Repls]] Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' New = '${1}[KILLED]' + +# On Windows a killed CLI leaves no bash "Killed" line. The kill lands right after the +# "Uploading bundle files" line, so it is followed directly by "Exit code: 1". +[[Repls]] +Old = '(Uploading bundle files to [^\n]+)\n\nExit code: 1' +New = "${1}\n[PROCESS_KILLED]\n\nExit code: [KILLED]" From 6bdb8f0e0ed1af955d0b0cb77332d4330fbd7787 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 15:47:19 +0100 Subject: [PATCH 21/21] direct: record volume_id and schema_id in the resource state instead of stable_id The internal ID now lives in the volume and schema state (VolumeState, SchemaState), set from the API responses and copied by RemapState; the framework reads it back through a new optional InternalID method. This replaces the stable_id field on the state entry and in the DMS envelope. A new drop_changes lifecycle rule leaves the always-different volume_id and schema_id (config never sets them) out of the plan. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 5 +- .../bundle/dms/declined-deploy/output.txt | 8 +-- .../dms/deployment-metadata-change/output.txt | 2 +- acceptance/bundle/dms/empty-grants/output.txt | 8 +-- .../bundle/dms/failed-recreate/output.txt | 8 +-- .../operation-upload-fails-once/output.txt | 14 ++-- .../bundle/dms/successful-recreate/output.txt | 16 ++--- .../bundle/refschema/output/schemas.txt | 2 +- .../bundle/refschema/output/volumes.txt | 2 +- .../non_existent_field/out.deploy.txt | 2 +- .../non_existent_field/out.plan.txt | 2 +- .../schemas/recreate_out_of_band/output.txt | 16 ++--- .../schemas/recreate_out_of_band/script | 3 +- .../schemas/rename_interrupted/output.txt | 2 +- .../schemas/rename_interrupted/script | 2 +- .../databricks.yml.tmpl | 2 + .../schemas/rename_with_volume_ref/output.txt | 8 +++ .../schemas/rename_with_volume_ref/script | 1 + .../volumes/recreate_out_of_band/output.txt | 2 +- .../volumes/recreate_out_of_band/script | 2 +- .../simple/out.requests.deploy.json | 6 +- bundle/direct/apply.go | 21 ++---- bundle/direct/bundle_plan.go | 32 +++++++--- bundle/direct/bundle_plan_test.go | 19 +++++- bundle/direct/dresources/README.md | 5 +- bundle/direct/dresources/adapter.go | 46 ++++++------- bundle/direct/dresources/config.go | 5 ++ bundle/direct/dresources/config_test.go | 1 + bundle/direct/dresources/configs/schemas.yml | 4 ++ bundle/direct/dresources/configs/volumes.yml | 4 ++ bundle/direct/dresources/schema.go | 64 ++++++++++++++----- bundle/direct/dresources/schema_test.go | 4 +- bundle/direct/dresources/volume.go | 55 ++++++++++++---- bundle/direct/dstate/dms.go | 2 - bundle/direct/dstate/dms_test.go | 13 ---- bundle/direct/dstate/state.go | 17 +---- bundle/direct/dstate/state_test.go | 17 ----- 37 files changed, 240 insertions(+), 182 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index e7f69ccda56..9badc1e4d5e 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -146,7 +146,6 @@ def get_resources(target): "id": resource.get("resource_id"), "state": envelope.get("state") or {}, "depends_on": envelope.get("depends_on") or [], - "stable_id": envelope.get("stable_id"), } page_token = listed.get("next_page_token") if not page_token: @@ -155,14 +154,12 @@ def get_resources(target): def get_recorded_state(target): """The recorded resources in the on-disk state file's `state` shape (resources. -> - {__id__, state, depends_on, stable_id}), so a recording run prints the same shape as a non-recording one.""" + {__id__, state, depends_on}), so a recording run prints the same shape as a non-recording one.""" state = {} for key, value in sorted(get_resources(target).items()): entry = {"__id__": value["id"], "state": value["state"]} if value["depends_on"]: entry["depends_on"] = value["depends_on"] - if value["stable_id"]: - entry["stable_id"] = value["stable_id"] state[f"resources.{key}"] = entry return state diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 5b6771f3d7b..dacc77db623 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -98,9 +98,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", "state": { "catalog_name": "main", - "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" - }, - "stable_id": "[UUID]" + "name": "dms_declined_deploy_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" + } } } } diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 8e96186b9ff..02cbb2e5705 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_metadata_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index ceeacc47b2b..5e169e86ebd 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -52,7 +52,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_empty_grants_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -170,9 +170,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "__id__": "main.dms_empty_grants_[UNIQUE_NAME]", "state": { "catalog_name": "main", - "name": "dms_empty_grants_[UNIQUE_NAME]" - }, - "stable_id": "[UUID]" + "name": "dms_empty_grants_[UNIQUE_NAME]", + "schema_id": "[UUID]" + } } } } diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 8be24682946..0d930eff2c0 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -100,9 +100,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "comment": "v1", - "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" - }, - "stable_id": "[UUID]" + "name": "dms_failed_recreate_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" + } } } } diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 8708782f930..6dbce4b3834 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -47,9 +47,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" - }, - "stable_id": "[UUID]" + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" + } } } } @@ -95,9 +95,9 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "state": { "catalog_name": "dms_other_[UNIQUE_NAME]", "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" - }, - "stable_id": "[UUID]" + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" + } } } } @@ -143,7 +143,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "error_message": "", "resource_id": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index dfbc8d82e8a..f6776520bc3 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -100,9 +100,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" - }, - "stable_id": "[UUID]" + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" + } } } } @@ -160,7 +160,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "error_message": "", "resource_id": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", "sequence_id": 1, - "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"},\"stable_id\":\"[UUID]\"}", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -213,9 +213,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "dms_other_[UNIQUE_NAME]", "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" - }, - "stable_id": "[UUID]" + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" + } } } } diff --git a/acceptance/bundle/refschema/output/schemas.txt b/acceptance/bundle/refschema/output/schemas.txt index 694d59c6211..566999fd664 100644 --- a/acceptance/bundle/refschema/output/schemas.txt +++ b/acceptance/bundle/refschema/output/schemas.txt @@ -23,7 +23,7 @@ resources.schemas.*.name string ALL resources.schemas.*.owner string REMOTE resources.schemas.*.properties map[string]string ALL resources.schemas.*.properties.* string ALL -resources.schemas.*.schema_id string REMOTE +resources.schemas.*.schema_id string REMOTE STATE resources.schemas.*.storage_location string REMOTE resources.schemas.*.storage_root string ALL resources.schemas.*.updated_at int64 REMOTE diff --git a/acceptance/bundle/refschema/output/volumes.txt b/acceptance/bundle/refschema/output/volumes.txt index 1c573ed1931..52e07594399 100644 --- a/acceptance/bundle/refschema/output/volumes.txt +++ b/acceptance/bundle/refschema/output/volumes.txt @@ -24,7 +24,7 @@ resources.volumes.*.storage_location string ALL resources.volumes.*.updated_at int64 REMOTE resources.volumes.*.updated_by string REMOTE resources.volumes.*.url string INPUT -resources.volumes.*.volume_id string REMOTE +resources.volumes.*.volume_id string REMOTE STATE resources.volumes.*.volume_path string INPUT resources.volumes.*.volume_type catalog.VolumeType ALL resources.volumes.*.grants.full_name string ALL diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt index 8da224f9751..d546194dba3 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt @@ -1,5 +1,5 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo +Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in dresources.VolumeState; non_existent: field "non_existent" not found in catalog.VolumeInfo Error: planning failed diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt b/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt index 6a2b5d1157c..232d9f0b3d4 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt @@ -1,4 +1,4 @@ -Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo +Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in dresources.VolumeState; non_existent: field "non_existent" not found in catalog.VolumeInfo Error: planning failed diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt index f2520fb4368..f85cf6c5f62 100644 --- a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt @@ -15,14 +15,14 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_state.py { - "resources.schemas.schema1": { - "__id__": "main.test-schema-[UNIQUE_NAME]", - "stable_id": "[SCHEMA_ID]" - }, - "resources.volumes.volume1": { - "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", - "stable_id": "[VOLUME_ID]" - } + "__id__": "main.test-schema-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "volume_id": "[VOLUME_ID]" } >>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/script b/acceptance/bundle/resources/schemas/recreate_out_of_band/script index 2d0997ea594..cadca8544af 100644 --- a/acceptance/bundle/resources/schemas/recreate_out_of_band/script +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/script @@ -32,7 +32,8 @@ print_requests.py //unity-catalog &> LOG.recreate_requests title "With the config renamed to match, the plan finds a schema and a volume at the new names, but their ids differ, so it plans creates" trace update_file.py databricks.yml test-schema- test-schema-renamed- -trace print_state.py | jq '.state | map_values({__id__, stable_id})' +trace print_state.py | jq '.state["resources.schemas.schema1"] | {__id__, schema_id: .state.schema_id}' +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' plan diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt index b2a40eebbfe..f2b97b7ae27 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -33,7 +33,7 @@ Exit code: [KILLED] >>> print_state.py { "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", - "stable_id": "[VOLUME_ID]" + "volume_id": "[VOLUME_ID]" } >>> [CLI] bundle plan --output json diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/script b/acceptance/bundle/resources/schemas/rename_interrupted/script index 196ee5d5faa..76ee184840a 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/script +++ b/acceptance/bundle/resources/schemas/rename_interrupted/script @@ -24,7 +24,7 @@ trace errcode $CLI bundle deploy -q trace print_requests.py //unity-catalog title "The volume is gone from its recorded path, so the plan looks for it at its path in config" -trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, stable_id}' +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' title "The next deploy updates the volume at its new path" diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl index 2624f14c7cf..bddda4baff0 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl @@ -19,3 +19,5 @@ resources: default: ${resources.volumes.volume1.full_name} - name: volume_id default: ${resources.volumes.volume1.id} + - name: volume_uuid + default: ${resources.volumes.volume1.volume_id} diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt index 47697e156fd..ceb77a34e65 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt @@ -17,6 +17,10 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged { "default": "main.test-schema-[UNIQUE_NAME].myvolume", "name": "volume_id" + }, + { + "default": "[VOLUME_ID]", + "name": "volume_uuid" } ] @@ -60,6 +64,10 @@ Resources: 0 created, 3 changed, 0 deleted, 0 unchanged { "default": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", "name": "volume_id" + }, + { + "default": "[VOLUME_ID]", + "name": "volume_uuid" } ] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/script b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script index 0b1627161b3..6418f29011d 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume_ref/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script @@ -13,6 +13,7 @@ trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests JOB_ID=$($CLI bundle summary --output json | jq -r .resources.jobs.job1.id) add_repl "$JOB_ID" JOB_ID +add_repl "$($CLI volumes read "main.test-schema-${UNIQUE_NAME}.myvolume" | jq -r .volume_id)" VOLUME_ID trace $CLI jobs get "$JOB_ID" | jq .settings.parameters title "Rename the schema" diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt index 8ebe0b53337..22819d47ef8 100644 --- a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt @@ -16,7 +16,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_state.py { "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", - "stable_id": "[VOLUME_ID]" + "volume_id": "[VOLUME_ID]" } >>> [CLI] volumes read main.test-schema-[UNIQUE_NAME].myvolume2 diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/script b/acceptance/bundle/resources/volumes/recreate_out_of_band/script index ce1a0dacf83..eaaa5d0268a 100644 --- a/acceptance/bundle/resources/volumes/recreate_out_of_band/script +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/script @@ -29,6 +29,6 @@ print_requests.py //unity-catalog &> LOG.recreate_requests title "With the config renamed to match, the plan finds a volume at the new name, but its volume_id differs, so it plans a create" trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" -trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, stable_id}' +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' trace $CLI volumes read "$SCHEMA.myvolume2" | jq '{full_name, volume_id}' plan diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.json index 569fd9a0a65..e31832f7f4a 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.json @@ -249,9 +249,9 @@ "__id__": "mycatalog.myschema", "state": { "catalog_name": "mycatalog", - "name": "myschema" - }, - "stable_id": "[UUID]" + "name": "myschema", + "schema_id": "[UUID]" + } } } } diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 84274ab56fc..7c9f12da677 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -89,7 +89,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, d.Adapter.StableID(d.RemoteState)) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -187,7 +187,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn, d.stableID(planEntry.RemoteState)) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -232,7 +232,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, d.stableID(planEntry.RemoteState)) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -325,7 +325,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn, d.stableID(entry.RemoteState)) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -337,7 +337,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // with content hashes, see dresources.CompactState) before persisting it. Fields already // smaller than a hash placeholder are persisted as is. Sensitive fields are replaced by a // placeholder value so secrets are never written to disk in plaintext. -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, stableID string) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), state) if err != nil { return fmt.Errorf("compacting state: %w", err) @@ -345,16 +345,7 @@ func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentSta if err := zeroSensitiveFields(d.Adapter, compacted); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveStateWithStableID(ctx, d.ResourceKey, newID, compacted, dependsOn, stableID) -} - -// stableID returns the stable ID (see dstate.ResourceEntry) of the object being saved: from the -// remote state the operation returned, else from planRemote, the same object as the plan read it. -func (d *DeploymentUnit) stableID(planRemote any) string { - if id := d.Adapter.StableID(d.RemoteState); id != "" { - return id - } - return d.Adapter.StableID(planRemote) + return db.SaveState(ctx, d.ResourceKey, newID, compacted, dependsOn) } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index b35c4e32441..64b88bd7205 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -335,7 +335,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } } if remoteState == nil { - entry.ID, remoteState, err = findMoved(ctx, adapter, sv, dbentry) + entry.ID, remoteState, err = findMoved(ctx, adapter, sv, dbentry.ID, savedState) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false @@ -501,6 +501,13 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change return err } + if cfg != nil { + if _, ok := findMatchingRule(path, cfg.DropChanges); ok { + toDrop = append(toDrop, pathString) + continue + } + } + // RemoteAlreadySet only holds when ch.Remote is a real remote value we can compare // against ch.New. Skip it for fields whose remote value is fabricated and thus // meaningless: declared ignore_remote_changes (present in RemoteType but read back @@ -742,15 +749,19 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. // findMoved looks for a resource missing from its recorded ID at the ID its config gives it, where a // rename can have moved it: the resource or its parent renamed out of band, or by a deploy that -// stopped before re-keying it. What is found there is taken only if its stable ID matches the -// recorded one, so an unrelated object with the same name is never adopted; the create then fails +// stopped before re-keying it. What is found there is taken only if its internal ID matches the one +// in savedState, so an unrelated object with the same name is never adopted; the create then fails // on it instead. -func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.StructVar, dbentry dstate.ResourceEntry) (string, any, error) { - if !adapter.HasStableID() || !adapter.HasDoUpdateWithID() || dbentry.StableID == "" || hasUnresolvedIDField(adapter, sv) { +func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.StructVar, storedID string, savedState any) (string, any, error) { + if !adapter.HasInternalID() || !adapter.HasDoUpdateWithID() || hasUnresolvedIDField(adapter, sv) { return "", nil, nil } + savedInternalID, err := adapter.InternalID(savedState) + if err != nil || savedInternalID == "" { + return "", nil, err + } id, err := adapter.IDFromState(sv.Value) - if err != nil || id == dbentry.ID { + if err != nil || id == storedID { return "", nil, err } remote, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { @@ -762,8 +773,13 @@ func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.S if err != nil { return "", nil, fmt.Errorf("reading id=%q: %w", id, err) } - if adapter.StableID(remote) != dbentry.StableID { - return "", nil, nil + found, err := adapter.RemapState(remote) + if err != nil { + return "", nil, err + } + foundInternalID, err := adapter.InternalID(found) + if err != nil || foundInternalID != savedInternalID { + return "", nil, err } return id, remote, nil } diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index f50de986918..1780a13134e 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -55,8 +55,8 @@ func TestDynPathToStructPath(t *testing.T) { func TestExtractReferences_ExcludesReadonlyFields(t *testing.T) { adapters, err := dresources.InitAll(nil) require.NoError(t, err) - // The volume state type is catalog.CreateVolumeRequestContent, which has - // comment but not volume_path. + // The volume state type is VolumeState, which has comment but not + // volume_path. stateType := adapters["volumes"].StateType() const yml = ` @@ -853,3 +853,18 @@ func TestLadderBackendDefaultBeforeRemoteAddition(t *testing.T) { assert.Equal(t, deployplan.Skip, changes["enable_elastic_disk"].Action) assert.Equal(t, deployplan.ReasonBackendDefault, changes["enable_elastic_disk"].Reason) } + +// TestDropChanges: a drop_changes field (volumes' volume_id, which config never sets) leaves the +// plan entirely, while the volume's other changes stay. +func TestDropChanges(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + changes := deployplan.Changes{ + "volume_id": {Old: "uuid-1", New: nil, Remote: "uuid-1"}, + "comment": {Old: "a", New: "b", Remote: "a"}, + } + require.NoError(t, addPerFieldActions(t.Context(), adapters["volumes"], changes, nil, nil)) + assert.NotContains(t, changes, "volume_id") + assert.Equal(t, deployplan.Update, changes["comment"].Action) +} diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index 98f10f85228..b29498ca69b 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -27,6 +27,7 @@ Choose the right category: - `output_only` — the field is computed by the backend; the user never sets it - `input_only` — accepted on create/update but not returned by GET (e.g., write-only tokens, flags) - `managed` — managed by the cloud provider or platform, not by the user config + - **`drop_changes`**: Leave changes to this field out of the plan entirely. Use for fields that config never sets but the state records, like an internal ID recorded from the backend (see `InternalID`). - **`ignore_local_changes`**: Ignore changes the user makes to this field. Use for fields that cannot be updated via API — either they are immutable after creation or require a separate API that is not yet implemented. Must have a comment explaining why. - **`recreate_on_changes`**: Changing this field requires delete + create. Use for truly immutable fields (name, type, location). The reason should reference API docs or TF provider. - **`updatable_id_fields`**: Changing this field changes the resource's ID. Requires `DoUpdateWithID` to be implemented. @@ -47,9 +48,9 @@ For resources whose create or update is asynchronous (the resource is not immedi If the API may return a slice's elements in a different order between calls (e.g., `depends_on` in job tasks, `privileges` in grants), implement `KeyedSlices` to compare elements by a natural key rather than by index. Without this, every deploy after any reordering shows phantom diffs. -## Moved resources: IDFromState / StableID +## Moved resources: IDFromState / InternalID -If the ID is built from the resource's own fields (a UC full name) and the backend has an ID that a rename keeps (a volume's `volume_id`), implement `IDFromState` and `StableID` together, along with `DoUpdateWithID`. When nothing is at the recorded ID, the plan reads at the ID the config gives the resource and takes what it finds only if its stable ID matches the one saved in state. This covers a rename done out of band, or by a deploy that stopped before re-keying the resource. +If the ID is built from the resource's own fields (a UC full name) and the backend has its own ID that a rename keeps (a volume's `volume_id`), record that ID in the state type, set it from the API responses (and from `RemapState`), and implement `IDFromState` and `InternalID` together, along with `DoUpdateWithID`. When nothing is at the recorded ID, the plan reads at the ID the config gives the resource and takes what it finds only if its internal ID matches the one saved in state. This covers a rename done out of band, or by a deploy that stopped before re-keying the resource. ## Empty states: IsEmptyState diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 75c91cdc4d5..64a32906462 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -120,15 +120,15 @@ type IResource interface { IsGone(remoteState any) bool // [Optional] IDFromState returns the ID the state gives the resource, for resource types whose ID - // is built from their own fields (a UC full name). Implemented together with StableID. - // Example: func (*ResourceVolume) IDFromState(state *catalog.CreateVolumeRequestContent) string + // is built from their own fields (a UC full name). Implemented together with InternalID. + // Example: func (*ResourceVolume) IDFromState(state *VolumeState) string IDFromState(state any) string - // [Optional] StableID returns the backend's ID for the object that a rename keeps (a volume's - // volume_id). When the resource is not at its recorded ID, the plan looks for it at the ID - // IDFromState gives it, and takes what it finds only if StableID matches the recorded one. - // Example: func (*ResourceVolume) StableID(remote *catalog.VolumeInfo) string - StableID(remoteState any) string + // [Optional] InternalID returns the backend's own ID recorded in the state, which a rename keeps + // (a volume's volume_id). When the resource is not at its recorded ID, the plan looks for it at the + // ID IDFromState gives it, and takes what it finds only if the internal IDs match. + // Example: func (*ResourceVolume) InternalID(state *VolumeState) string + InternalID(state any) string } // Adapter wraps resource implementation, validates signatures and type consistency across methods @@ -153,7 +153,7 @@ type Adapter struct { doResize *calladapt.BoundCaller isGone *calladapt.BoundCaller idFromState *calladapt.BoundCaller - stableID *calladapt.BoundCaller + internalID *calladapt.BoundCaller resourceConfig *ResourceLifecycleConfig generatedResourceConfig *ResourceLifecycleConfig @@ -196,7 +196,7 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC overrideChangeDesc: nil, isGone: nil, idFromState: nil, - stableID: nil, + internalID: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), keyedSlices: nil, @@ -328,7 +328,7 @@ func (a *Adapter) initMethods(resource any) error { return err } - a.stableID, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "StableID") + a.internalID, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "InternalID") if err != nil { return err } @@ -427,12 +427,12 @@ func (a *Adapter) validate() error { validations = append(validations, "IsGone remoteState", a.isGone.InTypes[0], remoteType) } - if (a.idFromState == nil) != (a.stableID == nil) { - return errors.New("IDFromState and StableID must be implemented together") + if (a.idFromState == nil) != (a.internalID == nil) { + return errors.New("IDFromState and InternalID must be implemented together") } if a.idFromState != nil { validations = append(validations, "IDFromState state", a.idFromState.InTypes[0], stateType) - validations = append(validations, "StableID remoteState", a.stableID.InTypes[0], remoteType) + validations = append(validations, "InternalID state", a.internalID.InTypes[0], stateType) } if a.doUpdateWithID != nil { @@ -760,9 +760,9 @@ func (a *Adapter) IsGone(remoteState any) bool { return outs[0].(bool) } -// HasStableID reports whether the resource implements IDFromState and StableID. -func (a *Adapter) HasStableID() bool { - return a.stableID != nil +// HasInternalID reports whether the resource implements IDFromState and InternalID. +func (a *Adapter) HasInternalID() bool { + return a.internalID != nil } // IDFromState returns the ID the state gives the resource; see IResource.IDFromState. @@ -774,17 +774,13 @@ func (a *Adapter) IDFromState(state any) (string, error) { return outs[0].(string), nil } -// StableID returns the rename-proof ID of the object in remoteState; see IResource.StableID. -// It is empty for a resource type without one and for a nil remote state. -func (a *Adapter) StableID(remoteState any) string { - if a.stableID == nil || remoteState == nil || reflect.ValueOf(remoteState).IsNil() { - return "" - } - outs, err := a.stableID.Call(remoteState) +// InternalID returns the backend's own ID recorded in state; see IResource.InternalID. +func (a *Adapter) InternalID(state any) (string, error) { + outs, err := a.internalID.Call(state) if err != nil { - return "" + return "", err } - return outs[0].(string) + return outs[0].(string), nil } // prepareCallRequired prepares a call and ensures the method is found. diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 9b3ae9080b8..1cf0f41bb0f 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -68,6 +68,10 @@ type ResourceLifecycleConfig struct { // IgnoreLocalChanges: field patterns where local changes are ignored (can't be updated via API). IgnoreLocalChanges []FieldRule `yaml:"ignore_local_changes,omitempty"` + // DropChanges: field patterns whose changes are left out of the plan: fields that config never + // sets but the state records, such as an internal ID recorded from the backend. + DropChanges []FieldRule `yaml:"drop_changes,omitempty"` + // RecreateOnChanges: field patterns that trigger delete + create when changed. RecreateOnChanges []FieldRule `yaml:"recreate_on_changes,omitempty"` @@ -155,6 +159,7 @@ const ( var empty = ResourceLifecycleConfig{ IgnoreRemoteChanges: nil, IgnoreLocalChanges: nil, + DropChanges: nil, RecreateOnChanges: nil, ProvidedIDFields: nil, UpdatableIDFields: nil, diff --git a/bundle/direct/dresources/config_test.go b/bundle/direct/dresources/config_test.go index 53279eed6c6..d653d1ec695 100644 --- a/bundle/direct/dresources/config_test.go +++ b/bundle/direct/dresources/config_test.go @@ -91,6 +91,7 @@ func categoryRules(c ResourceLifecycleConfig) []struct { }{ {"ignore_remote_changes", c.IgnoreRemoteChanges}, {"ignore_local_changes", c.IgnoreLocalChanges}, + {"drop_changes", c.DropChanges}, {"recreate_on_changes", c.RecreateOnChanges}, {"provided_id_fields", c.ProvidedIDFields}, {"updatable_id_fields", c.UpdatableIDFields}, diff --git a/bundle/direct/dresources/configs/schemas.yml b/bundle/direct/dresources/configs/schemas.yml index 02f68e4afe9..c50e8d12b62 100644 --- a/bundle/direct/dresources/configs/schemas.yml +++ b/bundle/direct/dresources/configs/schemas.yml @@ -16,3 +16,7 @@ backend_defaults: # Without this, every subsequent plan produces an Update whose payload is empty, # and UC rejects it with "UpdateSchema Nothing to update". - field: properties['unity.catalog.managed.*.defaults.*'] +drop_changes: + # Recorded from the backend (see InternalID); config never sets it. + - field: schema_id + reason: internal_id diff --git a/bundle/direct/dresources/configs/volumes.yml b/bundle/direct/dresources/configs/volumes.yml index d4b68e95875..9a6a9906d5e 100644 --- a/bundle/direct/dresources/configs/volumes.yml +++ b/bundle/direct/dresources/configs/volumes.yml @@ -16,3 +16,7 @@ normalize_slash: # https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25 - field: storage_location reason: uc_strips_trailing_slash +drop_changes: + # Recorded from the backend (see InternalID); config never sets it. + - field: volume_id + reason: internal_id diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 6e1856aa421..250d44fb975 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -2,15 +2,33 @@ package dresources import ( "context" + "fmt" "slices" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// SchemaState is the schema's config plus its schema_id. The backend assigns schema_id and a +// rename keeps it, so recording it lets the plan recognize the schema under a new name (see +// InternalID). +type SchemaState struct { + catalog.CreateSchema + SchemaId string `json:"schema_id,omitempty"` +} + +func (s *SchemaState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s SchemaState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceSchema struct { client *databricks.WorkspaceClient } @@ -19,12 +37,15 @@ func (*ResourceSchema) New(client *databricks.WorkspaceClient) *ResourceSchema { return &ResourceSchema{client: client} } -func (*ResourceSchema) PrepareState(input *resources.Schema) *catalog.CreateSchema { - return &input.CreateSchema +func (*ResourceSchema) PrepareState(input *resources.Schema) *SchemaState { + return &SchemaState{ + CreateSchema: input.CreateSchema, + SchemaId: "", // Recorded from the backend when the schema is created or updated. + } } -func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *catalog.CreateSchema { - return &catalog.CreateSchema{ +func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *SchemaState { + return &SchemaState{ CatalogName: info.CatalogName, Comment: info.Comment, CustomMaxRetentionHours: info.CustomMaxRetentionHours, @@ -32,6 +53,7 @@ func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *catalog.CreateSchem Properties: info.Properties, StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateSchema](info.ForceSendFields), + SchemaId: info.SchemaId, } } @@ -39,11 +61,12 @@ func (r *ResourceSchema) DoRead(ctx context.Context, id string) (*catalog.Schema return r.client.Schemas.GetByFullName(ctx, id) } -func (r *ResourceSchema) DoCreate(ctx context.Context, config *catalog.CreateSchema) (string, *catalog.SchemaInfo, error) { - response, err := r.client.Schemas.Create(ctx, *config) +func (r *ResourceSchema) DoCreate(ctx context.Context, config *SchemaState) (string, *catalog.SchemaInfo, error) { + response, err := r.client.Schemas.Create(ctx, config.CreateSchema) if err != nil || response == nil { return "", nil, err } + config.SchemaId = response.SchemaId return response.FullName, response, nil } @@ -78,21 +101,22 @@ func makeUpdateSchema(id, newName string, config *catalog.CreateSchema) catalog. } // DoUpdate updates the schema in place and returns remote state. -func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (*catalog.SchemaInfo, error) { - response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(id, "", config)) - if err != nil { +func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *SchemaState, _ *PlanEntry) (*catalog.SchemaInfo, error) { + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(id, "", &config.CreateSchema)) + if err != nil || response == nil { return nil, err } - if response != nil && response.FullName != id { + if response.FullName != id { log.Warnf(ctx, "schemas: response contains unexpected full_name=%#v (expected %#v)", response.FullName, id) } + config.SchemaId = response.SchemaId return response, nil } // DoUpdateWithID renames the schema in place when the name changes; the ID is the full name, so it changes too. -func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateSchema, entry *PlanEntry) (string, *catalog.SchemaInfo, error) { +func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *SchemaState, entry *PlanEntry) (string, *catalog.SchemaInfo, error) { nameFromID, err := getNameFromID(id) if err != nil { return "", nil, err @@ -109,28 +133,34 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * if entry.Changes.OnlyParentRenamed() { // UC already moved the schema with its catalog; only the ID changes. + remote, ok := entry.RemoteState.(*catalog.SchemaInfo) + if !ok { + return "", nil, fmt.Errorf("internal error: unexpected remote state type %T", entry.RemoteState) + } + config.SchemaId = remote.SchemaId return fullName, nil, nil } - response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, config)) + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, &config.CreateSchema)) if err != nil || response == nil { return "", nil, err } + config.SchemaId = response.SchemaId return response.FullName, response, nil } // IDFromState returns the full name, which is the ID. -func (*ResourceSchema) IDFromState(state *catalog.CreateSchema) string { +func (*ResourceSchema) IDFromState(state *SchemaState) string { return state.CatalogName + "." + state.Name } -// StableID returns the schema_id, which a rename keeps. -func (*ResourceSchema) StableID(remote *catalog.SchemaInfo) string { - return remote.SchemaId +// InternalID returns the schema_id. +func (*ResourceSchema) InternalID(state *SchemaState) string { + return state.SchemaId } -func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *catalog.CreateSchema) error { +func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *SchemaState) error { return r.client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{ FullName: id, Force: true, diff --git a/bundle/direct/dresources/schema_test.go b/bundle/direct/dresources/schema_test.go index ee6397afc6e..5c486cab1f1 100644 --- a/bundle/direct/dresources/schema_test.go +++ b/bundle/direct/dresources/schema_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "testing" - "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -15,13 +14,14 @@ func TestResourceSchema_DoUpdate_WithUnsupportedForceSendFields(t *testing.T) { adapter := (*ResourceSchema)(nil).New(client) ctx := t.Context() - config := &catalog.CreateSchema{ + config := &SchemaState{ CatalogName: "main", Name: "test_schema", Comment: "original comment", Properties: map[string]string{"key": "value"}, StorageRoot: "", ForceSendFields: nil, + SchemaId: "", } id, _, err := adapter.DoCreate(ctx, config) diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 06674d45f96..5f9f9ccefca 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -10,9 +10,26 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// VolumeState is the volume's config plus its volume_id. The backend assigns volume_id and a +// rename keeps it, so recording it lets the plan recognize the volume under a new name (see +// InternalID). +type VolumeState struct { + catalog.CreateVolumeRequestContent + VolumeId string `json:"volume_id,omitempty"` +} + +func (s *VolumeState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s VolumeState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceVolume struct { client *databricks.WorkspaceClient } @@ -21,12 +38,15 @@ func (*ResourceVolume) New(client *databricks.WorkspaceClient) *ResourceVolume { return &ResourceVolume{client: client} } -func (*ResourceVolume) PrepareState(input *resources.Volume) *catalog.CreateVolumeRequestContent { - return &input.CreateVolumeRequestContent +func (*ResourceVolume) PrepareState(input *resources.Volume) *VolumeState { + return &VolumeState{ + CreateVolumeRequestContent: input.CreateVolumeRequestContent, + VolumeId: "", // Recorded from the backend when the volume is created or updated. + } } -func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *catalog.CreateVolumeRequestContent { - return &catalog.CreateVolumeRequestContent{ +func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *VolumeState { + return &VolumeState{ CatalogName: info.CatalogName, Comment: info.Comment, Name: info.Name, @@ -34,6 +54,7 @@ func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *catalog.CreateVolum StorageLocation: info.StorageLocation, VolumeType: info.VolumeType, ForceSendFields: utils.FilterFields[catalog.CreateVolumeRequestContent](info.ForceSendFields), + VolumeId: info.VolumeId, } } @@ -41,18 +62,19 @@ func (r *ResourceVolume) DoRead(ctx context.Context, id string) (*catalog.Volume return r.client.Volumes.ReadByName(ctx, id) } -func (r *ResourceVolume) DoCreate(ctx context.Context, config *catalog.CreateVolumeRequestContent) (string, *catalog.VolumeInfo, error) { - response, err := r.client.Volumes.Create(ctx, *config) +func (r *ResourceVolume) DoCreate(ctx context.Context, config *VolumeState) (string, *catalog.VolumeInfo, error) { + response, err := r.client.Volumes.Create(ctx, config.CreateVolumeRequestContent) if err != nil { return "", nil, err } + config.VolumeId = response.VolumeId return response.FullName, response, nil } // See schemaForceSend. Verified against a real workspace: {"comment": ""} clears it. var volumeForceSend = []string{"Comment"} -func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ *PlanEntry) (*catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *VolumeState, _ *PlanEntry) (*catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: id, @@ -80,10 +102,11 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalo log.Warnf(ctx, "volumes: response contains unexpected full_name=%#v (expected %#v)", response.FullName, id) } + config.VolumeId = response.VolumeId return response, err } -func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, entry *PlanEntry) (string, *catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *VolumeState, entry *PlanEntry) (string, *catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: "", // Initialized below @@ -106,6 +129,11 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * if entry.Changes.OnlyParentRenamed() { // UC already moved the volume with its parent; only the ID changes. + remote, ok := entry.RemoteState.(*catalog.VolumeInfo) + if !ok { + return "", nil, fmt.Errorf("internal error: unexpected remote state type %T", entry.RemoteState) + } + config.VolumeId = remote.VolumeId return updateRequest.Name, nil, nil } @@ -118,20 +146,21 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * return "", nil, err } + config.VolumeId = response.VolumeId return response.FullName, response, nil } // IDFromState returns the full name, which is the ID. -func (*ResourceVolume) IDFromState(state *catalog.CreateVolumeRequestContent) string { +func (*ResourceVolume) IDFromState(state *VolumeState) string { return state.CatalogName + "." + state.SchemaName + "." + state.Name } -// StableID returns the volume_id, which a rename keeps. -func (*ResourceVolume) StableID(remote *catalog.VolumeInfo) string { - return remote.VolumeId +// InternalID returns the volume_id. +func (*ResourceVolume) InternalID(state *VolumeState) string { + return state.VolumeId } -func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *catalog.CreateVolumeRequestContent) error { +func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *VolumeState) error { return r.client.Volumes.DeleteByName(ctx, id) } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 4842b8de10b..57e10b8d4b4 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -14,7 +14,6 @@ import ( type RecordedState struct { State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` - StableID string `json:"stable_id,omitempty"` } // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS @@ -58,6 +57,5 @@ func stateEntry(res dms.Resource) (ResourceEntry, error) { ID: res.ID, State: recorded.State, DependsOn: recorded.DependsOn, - StableID: recorded.StableID, }, nil } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 0dc86caf879..ab4e1bc017f 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -44,19 +44,6 @@ func TestApplyDMSState(t *testing.T) { "resources.pipelines.bar": {ID: "456"}, }, }, - { - name: "stable_id comes back from the envelope", - recorded: []dms.Resource{ - {Key: "resources.schemas.foo", ID: "main.foo", State: `{"state":{"name":"foo"},"stable_id":"uuid-1"}`}, - }, - want: map[string]ResourceEntry{ - "resources.schemas.foo": { - ID: "main.foo", - State: json.RawMessage(`{"name":"foo"}`), - StableID: "uuid-1", - }, - }, - }, { // Nil is what a list holding nothing returns, and it means a successful deploy of // nothing rather than missing data. diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ecb0cb75ed6..384246913f4 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -168,11 +168,6 @@ type ResourceEntry struct { ID string `json:"__id__"` State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` - - // StableID is the backend's ID for the object that survives a rename (a volume's volume_id), - // for resource types that have one. When the object is not at ID any more, the plan looks for - // it at the ID its config gives it and takes what it finds there only if StableID matches. - StableID string `json:"stable_id,omitempty"` } type WALEntry struct { @@ -279,7 +274,7 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e var recorded json.RawMessage if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { var err error - recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn, StableID: entry.StableID}) + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn}) if err != nil { // Nothing the caller can act on, so record the failure without the state. recorded = nil @@ -305,11 +300,6 @@ func NewDatabase(lineage string, serial int) Database { // SaveState records the resource's state after an operation was applied to it. func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { - return db.SaveStateWithStableID(ctx, key, newID, state, dependsOn, "") -} - -// SaveStateWithStableID is SaveState that also records the object's stable ID (see ResourceEntry). -func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, stableID string) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -326,7 +316,6 @@ func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID ID: newID, State: json.RawMessage(jsonMessage), DependsOn: dependsOn, - StableID: stableID, } switch db.mode { @@ -340,7 +329,7 @@ func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, StableID: stableID}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) if err != nil { return err } @@ -355,7 +344,7 @@ func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID } db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, StableID: stableID}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) if err != nil { return err } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 6330a5d99e4..c5bfea2783b 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -38,23 +38,6 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { mustFinalize(t, &db2) } -func TestSaveStateWithStableIDRoundTrip(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - require.NoError(t, db.SaveStateWithStableID(t.Context(), "schemas.foo", "main.foo", map[string]string{"name": "foo"}, nil, "uuid-1")) - mustFinalize(t, &db) - - var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) - entry, ok := db2.GetResourceEntry("schemas.foo") - require.True(t, ok) - assert.Equal(t, "main.foo", entry.ID) - assert.Equal(t, "uuid-1", entry.StableID) - mustFinalize(t, &db2) -} - func TestFinalizeWithoutResourceWrites(t *testing.T) { for _, mode := range []Mode{WritingFile, WritingDeploymentHistoryShadow} { path := filepath.Join(t.TempDir(), "state.json")