MaterializationSet in DAB: Test the recreate approval for mutating a materialization set member's membership key field

acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl

@@ -20,8 +20,8 @@ resources:
         - name: id
       timeseries_column:
         name: timestamp
-    # Declared so a later phase can add a member for it, or rename the set's member to it. A member
-    # names a feature that must already exist.
+    # Declared so a later phase can add a member for it, or change the feature name the set's member
+    # references from feata's to featb's. A member can only reference a feature that already exists.
     featb:
       full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME
       source:

acceptance/bundle/resources/materialization_sets/immutable/derived-trigger.yml.tmpl

@@ -0,0 +1,50 @@
+bundle:
+  name: test-matset-immutable-$UNIQUE_NAME
+
+resources:
+  features:
+    feata:
+      full_name: $CATALOG_NAME.$SCHEMA_NAME.imma_$UNIQUE_NAME
+      source:
+        delta_table_source:
+          full_name: $TABLE_NAME
+      function:
+        aggregation_function:
+          count_function:
+            input: value
+          time_window:
+            sliding:
+              window_duration: 604800s
+              slide_duration: 86400s
+      entities:
+        - name: id
+      timeseries_column:
+        name: timestamp
+    featb:
+      full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME
+      source:
+        delta_table_source:
+          full_name: $TABLE_NAME
+      function:
+        aggregation_function:
+          count_function:
+            input: value
+          time_window:
+            sliding:
+              window_duration: 604800s
+              slide_duration: 86400s
+      entities:
+        - name: id
+      timeseries_column:
+        name: timestamp
+
+  materialization_sets:
+    set1:
+      materialized_features:
+        - feature_name: ${resources.features.featb.full_name}
+          offline_store_config:
+            catalog_name: $CATALOG_NAME
+            schema_name: $SCHEMA_NAME
+            table_name_prefix: imm_${UNIQUE_NAME}_off
+          cron_schedule_trigger:
+            mode: DERIVED

acceptance/bundle/resources/materialization_sets/immutable/renamed-member.yml.tmpl → acceptance/bundle/resources/materialization_sets/immutable/member-references-featb.yml.tmpl

@@ -20,8 +20,6 @@ resources:
         - name: id
       timeseries_column:
         name: timestamp
-    # Declared so a later phase can add a member for it, or rename the set's member to it. A member
-    # names a feature that must already exist.
     featb:
       full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME
       source:
@@ -43,6 +41,7 @@ resources:
   materialization_sets:
     set1:
       materialized_features:
+        # Unlike the earlier phases, the member references featb's feature name instead of feata's.
         - feature_name: ${resources.features.featb.full_name}
           offline_store_config:
             catalog_name: $CATALOG_NAME

acceptance/bundle/resources/materialization_sets/immutable/output.txt

@@ -14,7 +14,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immuta
 Created features.feata
 Created features.featb
 Created materialization_sets.set1
-Files: 4 uploaded, 0 deleted
+Files: 5 uploaded, 0 deleted
 Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
 
 === A member field the endpoint cannot alter is caught by the planner as a recreate, so the
@@ -85,12 +85,87 @@ Updated materialization_sets.set1
 Files: 2 uploaded, 0 deleted
 Resources: 0 created, 1 changed, 0 deleted, 2 unchanged
 
-=== A member whose feature name changes reads as one member removed and another added, so it
-=== plans a recreate as well
+=== Changing the feature name the set's materialized feature member references, from feata's to
+=== featb's, reads as one member removed and another added, so it plans a recreate as well
 >>> [CLI] bundle plan -o json
 Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed
 
 >>> set_action
 {
   "resources.materialization_sets.set1": "recreate"
 }
+
+=== That recreate needs approval too, and with it the set is rebuilt with its member referencing
+=== featb's feature name
+>>> musterr [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files...
+Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed
+
+This action will result in the deletion or recreation of the following materialization sets.
+Every materialized feature in a recreated set is dropped and recomputed from scratch, including
+the members whose configuration did not change:
+  recreate resources.materialization_sets.set1
+Error: the deployment requires destructive actions, but the current console does not support prompting.
+Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed.
+To proceed, use --auto-approve after reviewing the plan above.
+
+Files: 2 uploaded, 0 deleted
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files...
+Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed
+
+This action will result in the deletion or recreation of the following materialization sets.
+Every materialized feature in a recreated set is dropped and recomputed from scratch, including
+the members whose configuration did not change:
+  recreate resources.materialization_sets.set1
+Recreated materialization_sets.set1
+Files: 0 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 1 deleted, 2 unchanged
+
+>>> [CLI] bundle plan -o json
+
+>>> set_action
+{
+  "resources.materialization_sets.set1": "skip"
+}
+
+=== Switching the trigger from a manual cron expression to a DERIVED schedule is a recreate as
+=== well, and the deploy asks for approval before rebuilding the set
+>>> [CLI] bundle plan -o json
+
+>>> set_action
+{
+  "resources.materialization_sets.set1": "recreate"
+}
+
+>>> musterr [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files...
+
+This action will result in the deletion or recreation of the following materialization sets.
+Every materialized feature in a recreated set is dropped and recomputed from scratch, including
+the members whose configuration did not change:
+  recreate resources.materialization_sets.set1
+Error: the deployment requires destructive actions, but the current console does not support prompting.
+Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed.
+To proceed, use --auto-approve after reviewing the plan above.
+
+Files: 2 uploaded, 0 deleted
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files...
+
+This action will result in the deletion or recreation of the following materialization sets.
+Every materialized feature in a recreated set is dropped and recomputed from scratch, including
+the members whose configuration did not change:
+  recreate resources.materialization_sets.set1
+Recreated materialization_sets.set1
+Files: 0 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 1 deleted, 2 unchanged
+
+>>> [CLI] bundle plan -o json
+
+>>> set_action
+{
+  "resources.materialization_sets.set1": "skip"
+}

acceptance/bundle/resources/materialization_sets/immutable/script

@@ -53,9 +53,26 @@ trace $CLI bundle plan -o json > out.plan.json
 trace set_action
 trace $CLI bundle deploy --auto-approve
 
-title "A member whose feature name changes reads as one member removed and another added, so it"
-title "plans a recreate as well"
-render renamed-member.yml.tmpl
+title "Changing the feature name the set's materialized feature member references, from feata's to"
+title "featb's, reads as one member removed and another added, so it plans a recreate as well"
+render member-references-featb.yml.tmpl
+trace $CLI bundle plan -o json > out.plan.json
+trace set_action
+
+title "That recreate needs approval too, and with it the set is rebuilt with its member referencing"
+title "featb's feature name"
+trace musterr $CLI bundle deploy
+trace $CLI bundle deploy --auto-approve
+trace $CLI bundle plan -o json > out.plan.json
+trace set_action
+
+title "Switching the trigger from a manual cron expression to a DERIVED schedule is a recreate as"
+title "well, and the deploy asks for approval before rebuilding the set"
+render derived-trigger.yml.tmpl
+trace $CLI bundle plan -o json > out.plan.json
+trace set_action
+trace musterr $CLI bundle deploy
+trace $CLI bundle deploy --auto-approve
 trace $CLI bundle plan -o json > out.plan.json
 trace set_action