diff --git a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl index 32b9e9f20c7..b61e970becd 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl @@ -20,8 +20,8 @@ resources: - name: id timeseries_column: name: timestamp - # Declared so a later phase can add a member for it, or rename the set's member to it. A member - # names a feature that must already exist. + # Declared so a later phase can add a member for it, or change the feature name the set's member + # references from feata's to featb's. A member can only reference a feature that already exists. featb: full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME source: diff --git a/acceptance/bundle/resources/materialization_sets/immutable/derived-trigger.yml.tmpl b/acceptance/bundle/resources/materialization_sets/immutable/derived-trigger.yml.tmpl new file mode 100644 index 00000000000..00b248e5c2b --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/derived-trigger.yml.tmpl @@ -0,0 +1,50 @@ +bundle: + name: test-matset-immutable-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.imma_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED diff --git a/acceptance/bundle/resources/materialization_sets/immutable/renamed-member.yml.tmpl b/acceptance/bundle/resources/materialization_sets/immutable/member-references-featb.yml.tmpl similarity index 90% rename from acceptance/bundle/resources/materialization_sets/immutable/renamed-member.yml.tmpl rename to acceptance/bundle/resources/materialization_sets/immutable/member-references-featb.yml.tmpl index 8d4433af495..bd7a2be2b18 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/renamed-member.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/immutable/member-references-featb.yml.tmpl @@ -20,8 +20,6 @@ resources: - name: id timeseries_column: name: timestamp - # Declared so a later phase can add a member for it, or rename the set's member to it. A member - # names a feature that must already exist. featb: full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME source: @@ -43,6 +41,7 @@ resources: materialization_sets: set1: materialized_features: + # Unlike the earlier phases, the member references featb's feature name instead of feata's. - feature_name: ${resources.features.featb.full_name} offline_store_config: catalog_name: $CATALOG_NAME diff --git a/acceptance/bundle/resources/materialization_sets/immutable/output.txt b/acceptance/bundle/resources/materialization_sets/immutable/output.txt index 378b095433c..0af82616941 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/output.txt +++ b/acceptance/bundle/resources/materialization_sets/immutable/output.txt @@ -14,7 +14,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immuta Created features.feata Created features.featb Created materialization_sets.set1 -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === A member field the endpoint cannot alter is caught by the planner as a recreate, so the @@ -85,8 +85,8 @@ Updated materialization_sets.set1 Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 2 unchanged -=== A member whose feature name changes reads as one member removed and another added, so it -=== plans a recreate as well +=== Changing the feature name the set's materialized feature member references, from feata's to +=== featb's, reads as one member removed and another added, so it plans a recreate as well >>> [CLI] bundle plan -o json Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed @@ -94,3 +94,78 @@ Warn: planning resources.materialization_sets.set1: the member for feature "main { "resources.materialization_sets.set1": "recreate" } + +=== That recreate needs approval too, and with it the set is rebuilt with its member referencing +=== featb's feature name +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 2 uploaded, 0 deleted + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 2 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "skip" +} + +=== Switching the trigger from a manual cron expression to a DERIVED schedule is a recreate as +=== well, and the deploy asks for approval before rebuilding the set +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 2 uploaded, 0 deleted + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 2 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "skip" +} diff --git a/acceptance/bundle/resources/materialization_sets/immutable/script b/acceptance/bundle/resources/materialization_sets/immutable/script index ac82a56b273..24fcd5cf5c9 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/script +++ b/acceptance/bundle/resources/materialization_sets/immutable/script @@ -53,9 +53,26 @@ trace $CLI bundle plan -o json > out.plan.json trace set_action trace $CLI bundle deploy --auto-approve -title "A member whose feature name changes reads as one member removed and another added, so it" -title "plans a recreate as well" -render renamed-member.yml.tmpl +title "Changing the feature name the set's materialized feature member references, from feata's to" +title "featb's, reads as one member removed and another added, so it plans a recreate as well" +render member-references-featb.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "That recreate needs approval too, and with it the set is rebuilt with its member referencing" +title "featb's feature name" +trace musterr $CLI bundle deploy +trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "Switching the trigger from a manual cron expression to a DERIVED schedule is a recreate as" +title "well, and the deploy asks for approval before rebuilding the set" +render derived-trigger.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy +trace $CLI bundle deploy --auto-approve trace $CLI bundle plan -o json > out.plan.json trace set_action