From 42240b2a18e94ca839ae0a7cdd90a1e031b46358 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sun, 8 Nov 2026 03:39:11 +0000 Subject: [PATCH 1/3] Skip the lowercased read-back of a feature's full_name UC lowercases a feature's name, and create and get return it that way. Listing full_name in provided_id_fields skips that read-back difference, as it does for schemas and feature specs, so a mixed-case full_name no longer recreates the feature on every plan. The fake lowercases the name like the real endpoint. Co-authored-by: Isaac --- .../bundles/features-uppercase-name.md | 1 + .../features/uppercase-name/databricks.yml | 22 +++ .../features/uppercase-name/out.test.toml | 2 + .../features/uppercase-name/output.txt | 140 ++++++++++++++++++ .../resources/features/uppercase-name/script | 18 +++ .../features/uppercase-name/test.toml | 3 + bundle/direct/dresources/configs/features.yml | 4 + libs/testserver/feature.go | 3 + 8 files changed, 193 insertions(+) create mode 100644 .nextchanges/bundles/features-uppercase-name.md create mode 100644 acceptance/bundle/resources/features/uppercase-name/databricks.yml create mode 100644 acceptance/bundle/resources/features/uppercase-name/out.test.toml create mode 100644 acceptance/bundle/resources/features/uppercase-name/output.txt create mode 100644 acceptance/bundle/resources/features/uppercase-name/script create mode 100644 acceptance/bundle/resources/features/uppercase-name/test.toml diff --git a/.nextchanges/bundles/features-uppercase-name.md b/.nextchanges/bundles/features-uppercase-name.md new file mode 100644 index 00000000000..84db8577829 --- /dev/null +++ b/.nextchanges/bundles/features-uppercase-name.md @@ -0,0 +1 @@ +* Fix a feature whose `full_name` contains uppercase letters being recreated on every deploy. diff --git a/acceptance/bundle/resources/features/uppercase-name/databricks.yml b/acceptance/bundle/resources/features/uppercase-name/databricks.yml new file mode 100644 index 00000000000..4df2cf2bdbb --- /dev/null +++ b/acceptance/bundle/resources/features/uppercase-name/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: test-bundle + +resources: + features: + feature1: + full_name: main.myschema.MyFeature + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: account_id + timeseries_column: + name: event_ts diff --git a/acceptance/bundle/resources/features/uppercase-name/out.test.toml b/acceptance/bundle/resources/features/uppercase-name/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/features/uppercase-name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/features/uppercase-name/output.txt b/acceptance/bundle/resources/features/uppercase-name/output.txt new file mode 100644 index 00000000000..4805a368d43 --- /dev/null +++ b/acceptance/bundle/resources/features/uppercase-name/output.txt @@ -0,0 +1,140 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created features.feature1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //feature-engineering +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.MyFeature", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "sliding": { + "slide_duration": "86400s", + "window_duration": "604800s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} + +=== UC lowercases the name, so the id is lowercase +>>> [CLI] bundle summary -o json +"main.myschema.myfeature" + +=== A plan on an unchanged bundle skips the name instead of recreating the feature +>>> [CLI] bundle plan -o json +{ + "resources.features.feature1": { + "action": "skip", + "changes": { + "full_name": { + "action": "skip", + "reason": "id_field" + } + } + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Renaming the feature still recreates it +>>> update_file.py databricks.yml main.myschema.MyFeature main.myschema.MyOtherFeature + +>>> [CLI] bundle plan +recreate features.feature1 + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... + +This action will result in the deletion or recreation of the following Feature Views. +A Feature View that has materialized features in a materialization set cannot be deleted or +recreated. Remove its materialized features from the set in a separate deploy first, then delete +or recreate the Feature View. Deleting or recreating a Feature View that a FeatureSpec serves is +not blocked, and leaves the FeatureSpec stale: + recreate resources.features.feature1 +Recreated features.feature1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> print_requests.py //feature-engineering +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/features/main.myschema.myfeature" +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.MyOtherFeature", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "sliding": { + "slide_duration": "86400s", + "window_duration": "604800s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.feature1 + +This action will result in the deletion of the following Feature Views. +Deleting a Feature View that a FeatureSpec serves is not blocked, and leaves the FeatureSpec stale: + delete resources.features.feature1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +>>> print_requests.py //feature-engineering +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/features/main.myschema.myotherfeature" +} diff --git a/acceptance/bundle/resources/features/uppercase-name/script b/acceptance/bundle/resources/features/uppercase-name/script new file mode 100644 index 00000000000..8c92514110e --- /dev/null +++ b/acceptance/bundle/resources/features/uppercase-name/script @@ -0,0 +1,18 @@ +trace $CLI bundle deploy +trace print_requests.py //feature-engineering + +title "UC lowercases the name, so the id is lowercase" +trace $CLI bundle summary -o json | jq .resources.features.feature1.id + +title "A plan on an unchanged bundle skips the name instead of recreating the feature" +trace $CLI bundle plan -o json | jq '.plan | map_values({action, changes: (.changes // {} | map_values({action, reason}) | with_entries(select(.key == "full_name")))})' +trace $CLI bundle plan + +title "Renaming the feature still recreates it" +trace update_file.py databricks.yml "main.myschema.MyFeature" "main.myschema.MyOtherFeature" +trace $CLI bundle plan +trace $CLI bundle deploy --auto-approve +trace print_requests.py //feature-engineering + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //feature-engineering diff --git a/acceptance/bundle/resources/features/uppercase-name/test.toml b/acceptance/bundle/resources/features/uppercase-name/test.toml new file mode 100644 index 00000000000..bd4b66fe75e --- /dev/null +++ b/acceptance/bundle/resources/features/uppercase-name/test.toml @@ -0,0 +1,3 @@ +Ignore = [ + ".databricks", +] diff --git a/bundle/direct/dresources/configs/features.yml b/bundle/direct/dresources/configs/features.yml index c0cd511f2dd..e8c72ce50d1 100644 --- a/bundle/direct/dresources/configs/features.yml +++ b/bundle/direct/dresources/configs/features.yml @@ -1,3 +1,7 @@ +provided_id_fields: + # UC lowercases the name; remote returns "main.ml.myfeature" for config "main.ml.MyFeature". + - field: full_name + reason: id_field stable_output_fields: # name is the last component of full_name, which is immutable, so it cannot change on an # in-place update; see the stable_output_fields doc in config.go and ES-2202624. diff --git a/libs/testserver/feature.go b/libs/testserver/feature.go index fe3289e38d6..6ebdacc74f6 100644 --- a/libs/testserver/feature.go +++ b/libs/testserver/feature.go @@ -23,6 +23,9 @@ func (s *FakeWorkspace) FeaturesCreate(req Request) Response { } } + // UC normalizes feature names to lowercase. + feature.FullName = strings.ToLower(feature.FullName) + parts := strings.Split(feature.FullName, ".") if len(parts) != 3 { return Response{ From 27398b878f6bea056740605ff991a48e1fc01b2f Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sun, 8 Nov 2026 03:53:47 +0000 Subject: [PATCH 2/3] Link the features changelog entry to its PR Co-authored-by: Isaac --- .nextchanges/bundles/features-uppercase-name.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/features-uppercase-name.md b/.nextchanges/bundles/features-uppercase-name.md index 84db8577829..8e21fd42af4 100644 --- a/.nextchanges/bundles/features-uppercase-name.md +++ b/.nextchanges/bundles/features-uppercase-name.md @@ -1 +1 @@ -* Fix a feature whose `full_name` contains uppercase letters being recreated on every deploy. +* Fix a feature whose `full_name` contains uppercase letters being recreated on every deploy. ([#7398](https://github.com/databricks/cli/pull/7398)) From 5bd66aa0976ee2c32afead06fbe885b3df5b1054 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sun, 8 Nov 2026 08:37:44 +0000 Subject: [PATCH 3/3] Compare and send materialization set feature names in lowercase UC stores a feature's name lowercase and the service requires a set member to name its feature that way, while a member's feature_name keeps the config spelling, whether it is written out, a reference to a feature, or built from a schema resource's name. So a mixed-case member was rejected at create, and a set holding one planned a recreate on every deploy. - MaterializedFeatureMemberKey lowercases the feature name, so a mixed-case member pairs with its lowercase read-back. A name holding an unresolved reference keeps its case, since resource keys are case-sensitive. - OverrideChangeDesc skips the feature_name of a paired member, which can then differ only in case. - DoCreate and DoUpdate send a copy of the set with each member's feature name lowercased. - The test server's set create refuses a member that does not name its feature in lowercase, with the service's message. Co-authored-by: Isaac --- ...rialization-sets-uppercase-feature-name.md | 1 + .../uppercase-name/databricks.yml | 69 +++++ .../uppercase-name/out.test.toml | 2 + .../uppercase-name/output.txt | 237 ++++++++++++++++++ .../uppercase-name/script | 50 ++++ ...n_set_materialized_feature_members_test.go | 12 + .../config/resources/materialization_set.go | 10 +- .../resources/materialization_set_test.go | 20 ++ .../direct/dresources/materialization_set.go | 29 ++- .../dresources/materialization_set_test.go | 21 +- libs/testserver/materialization_set.go | 28 +++ 11 files changed, 472 insertions(+), 7 deletions(-) create mode 100644 .nextchanges/bundles/materialization-sets-uppercase-feature-name.md create mode 100644 acceptance/bundle/resources/materialization_sets/uppercase-name/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/uppercase-name/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/uppercase-name/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/uppercase-name/script diff --git a/.nextchanges/bundles/materialization-sets-uppercase-feature-name.md b/.nextchanges/bundles/materialization-sets-uppercase-feature-name.md new file mode 100644 index 00000000000..c18dfe0cd6e --- /dev/null +++ b/.nextchanges/bundles/materialization-sets-uppercase-feature-name.md @@ -0,0 +1 @@ +* Fix a materialization set member whose `feature_name` contains uppercase letters failing to deploy, or recreating the set on every deploy. ([#7398](https://github.com/databricks/cli/pull/7398)) diff --git a/acceptance/bundle/resources/materialization_sets/uppercase-name/databricks.yml b/acceptance/bundle/resources/materialization_sets/uppercase-name/databricks.yml new file mode 100644 index 00000000000..63f91e2d140 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/uppercase-name/databricks.yml @@ -0,0 +1,69 @@ +bundle: + name: test-bundle + +resources: + schemas: + myschema: + catalog_name: main + name: MySchema + + features: + # The schema part of each name becomes ${resources.schemas.myschema.name}, which resolves to + # MySchema. + feata: + full_name: main.MySchema.FeatA + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: account_id + timeseries_column: + name: event_ts + featb: + full_name: main.MySchema.FeatB + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: account_id + timeseries_column: + name: event_ts + + materialization_sets: + set1: + lifecycle: + depends_on: + - resource: features.featb + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: main.MySchema.FeatB + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/uppercase-name/out.test.toml b/acceptance/bundle/resources/materialization_sets/uppercase-name/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/uppercase-name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/uppercase-name/output.txt b/acceptance/bundle/resources/materialization_sets/uppercase-name/output.txt new file mode 100644 index 00000000000..637fcfcadc4 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/uppercase-name/output.txt @@ -0,0 +1,237 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created features.feata +Created features.featb +Created materialization_sets.set1 +Created schemas.myschema +Files: 1 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +=== Each feature is created under the schema's spelling +>>> print_requests.py --keep //feature-engineering/features +[ + "main.MySchema.FeatA", + "main.MySchema.FeatB" +] + +=== Each member names its feature in lowercase, from a reference or written out +>>> print_requests.py //feature-engineering/materialization-sets +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/materialization-sets", + "q": { + "request_id": "[UUID]" + }, + "body": { + "materialized_features": [ + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.myschema.feata", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" + } + }, + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.myschema.featb", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" + } + } + ] + } +} + +=== A plan on an unchanged bundle skips each member's feature name instead of recreating the set +>>> set_plan +{ + "action": "skip", + "changes": { + "materialized_features[member='feature_name=main.myschema.feata|is_streaming_mode=false'].feature_name": { + "action": "skip", + "reason": "feature_name_case_only" + }, + "materialized_features[member='feature_name=main.myschema.featb|is_streaming_mode=false'].feature_name": { + "action": "skip", + "reason": "feature_name_case_only" + } + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +=== Changing only the case of a written-out feature name is skipped +>>> update_file.py databricks.yml feature_name: main.MySchema.FeatB feature_name: main.myschema.featb + +>>> set_plan +{ + "action": "skip", + "changes": { + "materialized_features[member='feature_name=main.myschema.feata|is_streaming_mode=false'].feature_name": { + "action": "skip", + "reason": "feature_name_case_only" + }, + "materialized_features[member='feature_name=main.myschema.featb|is_streaming_mode=false'].feature_name": { + "action": "skip", + "reason": "feature_name_case_only" + } + } +} + +>>> update_file.py databricks.yml feature_name: main.myschema.featb feature_name: MAIN.MYSCHEMA.FEATB + +>>> set_plan +{ + "action": "skip", + "changes": { + "materialized_features[member='feature_name=main.myschema.feata|is_streaming_mode=false'].feature_name": { + "action": "skip", + "reason": "feature_name_case_only" + }, + "materialized_features[member='feature_name=main.myschema.featb|is_streaming_mode=false'].feature_name": { + "action": "skip", + "reason": "feature_name_case_only" + } + } +} + +=== Naming another feature still recreates the set +>>> update_file.py databricks.yml feature_name: MAIN.MYSCHEMA.FEATB feature_name: main.MySchema.FeatC + +>>> set_plan +Warn: planning resources.materialization_sets.set1: the member for feature "main.MySchema.FeatC" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed +{ + "action": "recreate", + "changes": { + "materialized_features[member='feature_name=main.myschema.feata|is_streaming_mode=false'].feature_name": { + "action": "skip", + "reason": "feature_name_case_only" + }, + "materialized_features[member='feature_name=main.myschema.featb|is_streaming_mode=false']": { + "action": "update", + "reason": "custom" + }, + "materialized_features[member='feature_name=main.myschema.featc|is_streaming_mode=false']": { + "action": "recreate", + "reason": "member_added" + } + } +} + +>>> update_file.py databricks.yml feature_name: main.MySchema.FeatC feature_name: main.MySchema.FeatB + +=== Removing a member updates the set, naming the remaining member's feature in lowercase +>>> update_file.py databricks.yml - feature_name: main.MySchema.FeatB + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +>>> print_requests.py //feature-engineering/materialization-sets +{ + "method": "PATCH", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "request_id": "[UUID]", + "update_mask": "*" + }, + "body": { + "materialization_set_id": "[UUID]", + "materialized_features": [ + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.myschema.feata", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" + } + } + ] + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +=== The service refuses a member that names its feature in other than lowercase +>>> musterr [CLI] feature-engineering create-materialization-set --json {"materialized_features": [{"feature_name": "main.MySchema.FeatA", "offline_store_config": {"catalog_name": "main", "schema_name": "myschema", "table_name_prefix": "direct"}, "cron_schedule_trigger": {"mode": "MANUAL", "cron_expression": "0 0 * * * ?"}}]} +Error: operation failed: [BAD_REQUEST] Materialized feature requests must include the resolved feature names: main.myschema.feata + +>>> print_requests.py //feature-engineering/materialization-sets +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/materialization-sets", + "q": { + "request_id": "[UUID]" + }, + "body": { + "materialized_features": [ + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.MySchema.FeatA", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "direct" + } + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.feata + delete resources.features.featb + delete resources.materialization_sets.set1 + delete resources.schemas.myschema + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.myschema + +This action will result in the deletion of the following Feature Views. +Deleting a Feature View that a FeatureSpec serves is not blocked, and leaves the FeatureSpec stale: + delete resources.features.feata + delete resources.features.featb + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 4 deleted + +>>> print_requests.py //feature-engineering/materialization-sets +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "request_id": "[UUID]" + } +} diff --git a/acceptance/bundle/resources/materialization_sets/uppercase-name/script b/acceptance/bundle/resources/materialization_sets/uppercase-name/script new file mode 100644 index 00000000000..3353c392543 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/uppercase-name/script @@ -0,0 +1,50 @@ +# The set's action, and the plan entries for each member and its feature name. +set_plan() { + $CLI bundle plan -o json | jq '.plan."resources.materialization_sets.set1" | {action, changes: (.changes // {} | with_entries(select(.key | test("(\\]|feature_name)$"))) | map_values({action, reason}))}' +} + +trace $CLI bundle deploy + +title "Each feature is created under the schema's spelling" +trace print_requests.py --keep //feature-engineering/features | jq -s 'map(.body.full_name) | sort' + +title "Each member names its feature in lowercase, from a reference or written out" +trace print_requests.py //feature-engineering/materialization-sets + +title "A plan on an unchanged bundle skips each member's feature name instead of recreating the set" +trace set_plan +trace $CLI bundle plan + +title "Changing only the case of a written-out feature name is skipped" +trace update_file.py databricks.yml "feature_name: main.MySchema.FeatB" "feature_name: main.myschema.featb" +trace set_plan +trace update_file.py databricks.yml "feature_name: main.myschema.featb" "feature_name: MAIN.MYSCHEMA.FEATB" +trace set_plan + +title "Naming another feature still recreates the set" +trace update_file.py databricks.yml "feature_name: MAIN.MYSCHEMA.FEATB" "feature_name: main.MySchema.FeatC" +trace set_plan +trace update_file.py databricks.yml "feature_name: main.MySchema.FeatC" "feature_name: main.MySchema.FeatB" + +title "Removing a member updates the set, naming the remaining member's feature in lowercase" +trace update_file.py databricks.yml " - feature_name: main.MySchema.FeatB + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline + cron_schedule_trigger: + mode: MANUAL + cron_expression: \"0 0 * * * ?\" +" "" +trace $CLI bundle deploy +trace print_requests.py //feature-engineering/materialization-sets +trace $CLI bundle plan + +title "The service refuses a member that names its feature in other than lowercase" +trace musterr $CLI feature-engineering create-materialization-set --json '{"materialized_features": [{"feature_name": "main.MySchema.FeatA", "offline_store_config": {"catalog_name": "main", "schema_name": "myschema", "table_name_prefix": "direct"}, "cron_schedule_trigger": {"mode": "MANUAL", "cron_expression": "0 0 * * * ?"}}]}' +trace print_requests.py //feature-engineering/materialization-sets + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //feature-engineering/materialization-sets + +rm -f out.requests.txt diff --git a/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members_test.go b/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members_test.go index a5f71b9b658..8738e1a9a4a 100644 --- a/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members_test.go +++ b/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members_test.go @@ -179,3 +179,15 @@ func TestNormalizeMaterializationSetMaterializedFeatureMembersNamesAFeatureOnceF require.Len(t, diags, 1) assert.Equal(t, duplicateMemberKeyError("'"+memberFeatureName+"'"), diags[0].Summary) } + +func TestNormalizeMaterializationSetMaterializedFeatureMembersRejectsTheSameMemberSpelledInOtherCase(t *testing.T) { + _, diags := normalizeSetDiags(t, &resources.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + offlineMember(memberFeatureName), + offlineMember("main.MySchema.Feat"), + }, + }) + + require.Len(t, diags, 1) + assert.Equal(t, duplicateMemberKeyError("'main.MySchema.Feat', '"+memberFeatureName+"'"), diags[0].Summary) +} diff --git a/bundle/config/resources/materialization_set.go b/bundle/config/resources/materialization_set.go index 6dc3072d0ff..3b6a8cd8457 100644 --- a/bundle/config/resources/materialization_set.go +++ b/bundle/config/resources/materialization_set.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -46,8 +47,15 @@ func escapeMemberKeyPart(part string) string { // // The server pairs members by the same components, so this encoding is the CLI's own and only the // component list has to agree with the service. +// +// A feature name holding an unresolved reference keeps its case, because resource keys are +// case-sensitive. func MaterializedFeatureMemberKey(member ml.MaterializedFeature) (string, string) { - parts := []string{"feature_name=" + escapeMemberKeyPart(member.FeatureName)} + featureName := member.FeatureName + if !dynvar.ContainsVariableReference(featureName) { + featureName = strings.ToLower(featureName) + } + parts := []string{"feature_name=" + escapeMemberKeyPart(featureName)} if member.OnlineStoreConfig != nil { parts = append(parts, "online_store_name="+escapeMemberKeyPart(member.OnlineStoreConfig.OnlineStoreName)) diff --git a/bundle/config/resources/materialization_set_test.go b/bundle/config/resources/materialization_set_test.go index aae15b9300e..8bdd7afa7eb 100644 --- a/bundle/config/resources/materialization_set_test.go +++ b/bundle/config/resources/materialization_set_test.go @@ -99,3 +99,23 @@ func TestMaterializationSetMemberKeySeparatesAbsentOnlineStoreFromEmptyName(t *t onlineWithoutName) assert.NotEqual(t, offline, onlineWithoutName) } + +func TestMaterializationSetMemberKeyIgnoresFeatureNameCase(t *testing.T) { + _, mixedCase := MaterializedFeatureMemberKey(ml.MaterializedFeature{FeatureName: "main.MySchema.Feat"}) + _, lowercase := MaterializedFeatureMemberKey(ml.MaterializedFeature{FeatureName: "main.myschema.feat"}) + + assert.Equal(t, "feature_name=main.myschema.feat|is_streaming_mode=false", mixedCase) + assert.Equal(t, lowercase, mixedCase) +} + +func TestMaterializationSetMemberKeyKeepsTheCaseOfAReference(t *testing.T) { + _, mixedCaseKey := MaterializedFeatureMemberKey(ml.MaterializedFeature{ + FeatureName: "${resources.features.Feat.full_name}", + }) + _, lowercaseKey := MaterializedFeatureMemberKey(ml.MaterializedFeature{ + FeatureName: "${resources.features.feat.full_name}", + }) + + assert.Equal(t, "feature_name=${resources.features.Feat.full_name}|is_streaming_mode=false", mixedCaseKey) + assert.NotEqual(t, lowercaseKey, mixedCaseKey) +} diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 345a942d110..ca3d8ad370e 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -39,6 +39,8 @@ var ( materializedFeaturesPattern = structpath.NewPatternDotString(nil, "materialized_features") materializedFeatureMember = structpath.NewPatternBracketStar(materializedFeaturesPattern) + materializedFeatureMemberFeatureName = structpath.NewPatternDotString(materializedFeatureMember, "feature_name") + // The destination fields the server accepts on write and never returns on read. The whole // offline block is input-only; online_store_name is returned, so it is deliberately absent. redactedDestinationPatterns = []*structpath.PatternNode{ @@ -76,9 +78,11 @@ func isAbsentRemotely(remote any) bool { // reasonMemberAdded marks a recreate caused by a configured member the deployed set lacks. const reasonMemberAdded = "member_added" +const reasonFeatureNameCaseOnly = "feature_name_case_only" + // OverrideChangeDesc plans a member removed from the set as an update and a member the deployed -// set lacks as a recreate, and suppresses the drift left behind by an ignore_remote_changes rule -// that cannot fire. +// set lacks as a recreate, skips a member's feature name that differs only in case, and suppresses +// the drift left behind by an ignore_remote_changes rule that cannot fire. func (*ResourceMaterializationSet) OverrideChangeDesc(ctx context.Context, path *structpath.PathNode, change *ChangeDesc, remote *ml.MaterializationSet) error { // The immutable field rules also match a whole member, materialized_features[member=...], and // the whole list, materialized_features, because the matcher truncates a longer pattern. The @@ -100,6 +104,14 @@ func (*ResourceMaterializationSet) OverrideChangeDesc(ctx context.Context, path return nil } + // A paired member's feature names differ only in case, because MaterializedFeatureMemberKey + // lowercases them. + if path.HasPatternPrefix(materializedFeatureMemberFeatureName) { + change.Action = deployplan.Skip + change.Reason = reasonFeatureNameCaseOnly + return nil + } + // The ignore_remote_changes suppression on a destination field cannot fire for this resource, // which keys its member slice on a composite key defined by materializationSetMemberKey. The // suppression fires only when the persisted value equals the configured @@ -150,6 +162,15 @@ func (r *ResourceMaterializationSet) DoRead(ctx context.Context, id string) (*ml }) } +// withLowercaseFeatureNames returns a copy of the set with each member's feature name lowercased. +func withLowercaseFeatureNames(set ml.MaterializationSet) ml.MaterializationSet { + set.MaterializedFeatures = slices.Clone(set.MaterializedFeatures) + for i := range set.MaterializedFeatures { + set.MaterializedFeatures[i].FeatureName = strings.ToLower(set.MaterializedFeatures[i].FeatureName) + } + return set +} + // DoCreate submits the set and waits for the long-running operation the create returns. The created // set arrives as that operation's response, which Wait reaches by polling GetOperation. func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.MaterializationSet) (string, *ml.MaterializationSet, error) { @@ -157,7 +178,7 @@ func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.Ma // returns the operation the first request with the same idempotency token created. Minted here // so the token the server sees is the one this call chose. operation, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ - MaterializationSet: *config, + MaterializationSet: withLowercaseFeatureNames(*config), RequestId: uuid.New().String(), ForceSendFields: nil, }) @@ -234,7 +255,7 @@ func materializationSetOperationTarget(operation materializationSetOperation) st // DoUpdate reconciles the set's membership and waits for the long-running operation the update // returns. The reconciled set arrives as that operation's response. func (r *ResourceMaterializationSet) DoUpdate(ctx context.Context, id string, config *ml.MaterializationSet, _ *PlanEntry) (*ml.MaterializationSet, error) { - update := *config + update := withLowercaseFeatureNames(*config) update.MaterializationSetId = id operation, err := r.client.FeatureEngineering.UpdateMaterializationSet(ctx, ml.UpdateMaterializationSetRequest{ diff --git a/bundle/direct/dresources/materialization_set_test.go b/bundle/direct/dresources/materialization_set_test.go index 9fc65af042f..d2c7e6cb6c6 100644 --- a/bundle/direct/dresources/materialization_set_test.go +++ b/bundle/direct/dresources/materialization_set_test.go @@ -130,7 +130,8 @@ func TestMaterializationSetOperationFailureWithoutFailedMembers(t *testing.T) { // TestMaterializationSetMembershipChangeAction enumerates every path shape the planner hands this // resource: a whole member and the whole member list are an update unless they hold a member the -// deployed set lacks, and every other path keeps the action the rules gave it. +// deployed set lacks, a member's feature name is skipped, and every other path keeps the action the +// rules gave it. func TestMaterializationSetMembershipChangeAction(t *testing.T) { const member = "materialized_features[member='feature_name=main.ns.feat|online_store_name=store|is_streaming_mode=false']" const streamingMember = "materialized_features[member='feature_name=main.ns.feat|online_store_name=store|is_streaming_mode=true']" @@ -146,6 +147,8 @@ func TestMaterializationSetMembershipChangeAction(t *testing.T) { turnedStreaming.StreamingMode = &ml.StreamingMode{} movedStore := deployed movedStore.OnlineStoreConfig = &ml.OnlineStoreConfig{OnlineStoreName: "other"} + mixedCase := deployed + mixedCase.FeatureName = "main.NS.Feat" remote := &ml.MaterializationSet{MaterializedFeatures: []ml.MaterializedFeature{deployed}} for _, test := range []struct { @@ -160,12 +163,13 @@ func TestMaterializationSetMembershipChangeAction(t *testing.T) { {"member whose trigger turned streaming", streamingMember, turnedStreaming, remote, deployplan.Recreate, reasonMemberAdded}, {"member whose online store changed", movedStoreMember, movedStore, remote, deployplan.Recreate, reasonMemberAdded}, {"member the deployed set has", member, deployed, remote, deployplan.Update, "immutable"}, + {"member the deployed set has, naming its feature in other case", member, mixedCase, remote, deployplan.Update, "immutable"}, {"member removed", member, nil, remote, deployplan.Update, "immutable"}, {"list holding a member the deployed set lacks", "materialized_features", []ml.MaterializedFeature{deployed, added}, remote, deployplan.Recreate, reasonMemberAdded}, {"list the deployed set has", "materialized_features", []ml.MaterializedFeature{deployed}, remote, deployplan.Update, "immutable"}, {"list emptied", "materialized_features", nil, remote, deployplan.Update, "immutable"}, {"set missing remotely", addedMember, added, nil, deployplan.Update, "immutable"}, - {"feature name field", member + ".feature_name", nil, remote, deployplan.Recreate, "immutable"}, + {"feature name field", member + ".feature_name", "main.NS.Feat", remote, deployplan.Skip, reasonFeatureNameCaseOnly}, {"trigger field", member + ".cron_schedule_trigger", nil, remote, deployplan.Recreate, "immutable"}, {"destination field", member + ".online_store_config.table_name_prefix", nil, remote, deployplan.Recreate, "immutable"}, {"set id", "materialization_set_id", nil, remote, deployplan.Recreate, "immutable"}, @@ -198,3 +202,16 @@ func TestMaterializationSetDestinationSuppressionStillReachesMemberFields(t *tes assert.Equal(t, deployplan.Skip, change.Action) assert.Equal(t, "input_only_not_returned", change.Reason) } + +func TestWithLowercaseFeatureNamesCopiesTheSet(t *testing.T) { + config := ml.MaterializationSet{MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.MySchema.Feat"}, + {FeatureName: "main.myschema.other"}, + }} + + sent := withLowercaseFeatureNames(config) + + assert.Equal(t, "main.myschema.feat", sent.MaterializedFeatures[0].FeatureName) + assert.Equal(t, "main.myschema.other", sent.MaterializedFeatures[1].FeatureName) + assert.Equal(t, "main.MySchema.Feat", config.MaterializedFeatures[0].FeatureName) +} diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 15a62858f2e..0cb9f75b700 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -130,6 +130,33 @@ func featureMaterializationDestination(members []ml.MaterializedFeature) string return offline + "|" + online } +// validateResolvedFeatureNames mirrors CreateMaterializationSet's rule that each member names its +// feature by its resolved name, which UC stores lowercase. +func validateResolvedFeatureNames(members []ml.MaterializedFeature) *materializationSetRefusal { + requestedFeatureNames := map[string]bool{} + for _, member := range members { + requestedFeatureNames[member.FeatureName] = true + } + + var missingResolvedFeatureNames []string + for featureName := range requestedFeatureNames { + resolvedFeatureName := strings.ToLower(featureName) + if !requestedFeatureNames[resolvedFeatureName] && + !slices.Contains(missingResolvedFeatureNames, resolvedFeatureName) { + missingResolvedFeatureNames = append(missingResolvedFeatureNames, resolvedFeatureName) + } + } + if len(missingResolvedFeatureNames) == 0 { + return nil + } + slices.Sort(missingResolvedFeatureNames) + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: "Materialized feature requests must include the resolved feature names: " + + strings.Join(missingResolvedFeatureNames, ", "), + } +} + // validateSharedMaterializationDestination mirrors the create-path rule that every feature in a // set asks for an identical destination pair. Because the pair carries the online config, a set // cannot mix a feature materialized to both stores with one materialized offline only. @@ -330,6 +357,7 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { // than a rejected request: the backend reaches these rules inside the batch create it calls // after reserving, and reports them by settling the record. for _, bad := range []*materializationSetRefusal{ + validateResolvedFeatureNames(set.MaterializedFeatures), s.validateBatchAndStreamingFeaturesNotMixed(set.MaterializedFeatures), validateSharedMaterializationDestination(set.MaterializedFeatures), s.validateSawtoothMaterializedFeatures(set.MaterializedFeatures),