diff --git a/.nextchanges/bundles/feature-specs.md b/.nextchanges/bundles/feature-specs.md new file mode 100644 index 00000000000..1cb9ae4bc25 --- /dev/null +++ b/.nextchanges/bundles/feature-specs.md @@ -0,0 +1 @@ +* Add support for the `feature_specs` resource type, including UC grants on a feature spec. ([#7364](https://github.com/databricks/cli/pull/7364)) diff --git a/acceptance/bin/create_feature_spec_fixtures.py b/acceptance/bin/create_feature_spec_fixtures.py new file mode 100755 index 00000000000..78fe9ed9b05 --- /dev/null +++ b/acceptance/bin/create_feature_spec_fixtures.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +""" +Creates the feature tables and Python UDFs that the feature_specs acceptance tests look up, in an +existing schema. Creating a feature spec resolves each of them in Unity Catalog. + +Usage: create_feature_spec_fixtures.py CATALOG.SCHEMA +""" + +import json +import os +import subprocess +import sys +import time + +schema_full_name = sys.argv[1] +cli = os.environ.get("CLI", "databricks") +warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] + +statements = [ + f"CREATE TABLE {schema_full_name}.user_features " + "(user_id INT NOT NULL, age INT, country STRING, CONSTRAINT user_features_pk PRIMARY KEY (user_id))", + f"CREATE TABLE {schema_full_name}.tenant_user_features " + "(tenant_id INT NOT NULL, user_id INT NOT NULL, age INT, country STRING, " + "CONSTRAINT tenant_user_features_pk PRIMARY KEY (tenant_id, user_id))", + f"CREATE FUNCTION {schema_full_name}.score(x INT) RETURNS DOUBLE LANGUAGE PYTHON AS $$ return float(x) $$", + f"CREATE FUNCTION {schema_full_name}.country_score(x INT, y STRING) RETURNS DOUBLE LANGUAGE PYTHON " + "AS $$ return float(x) + len(y) $$", +] + + +def call_api(*args): + result = subprocess.run([cli, "api", *args], capture_output=True, text=True) + if result.returncode != 0: + print(f"databricks api {' '.join(args)} failed: {result.stderr}", file=sys.stderr) + sys.exit(1) + return json.loads(result.stdout) + + +for statement in statements: + payload = json.dumps({"warehouse_id": warehouse_id, "statement": statement, "wait_timeout": "50s"}) + response = call_api("post", "/api/2.0/sql/statements/", "--json", payload) + # A warehouse that is still starting outlasts wait_timeout. + deadline = time.monotonic() + 600 + while response["status"]["state"] in ("PENDING", "RUNNING") and time.monotonic() < deadline: + time.sleep(5) + response = call_api("get", f"/api/2.0/sql/statements/{response['statement_id']}") + if response["status"]["state"] != "SUCCEEDED": + print(f"Statement {statement!r} ended in {response['status']}", file=sys.stderr) + sys.exit(1) diff --git a/acceptance/bundle/deployment/bind/feature_spec/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/feature_spec/databricks.yml.tmpl new file mode 100644 index 00000000000..d0f748a0acc --- /dev/null +++ b/acceptance/bundle/deployment/bind/feature_spec/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: test-bind-feature-spec-$UNIQUE_NAME + +resources: + feature_specs: + spec1: + name: $SCHEMA_FULL_NAME.boundspec + features: + - feature_lookup: + table: $SCHEMA_FULL_NAME.user_features + lookup_keys: [user_id] + feature_names: [age] diff --git a/acceptance/bundle/deployment/bind/feature_spec/out.test.toml b/acceptance/bundle/deployment/bind/feature_spec/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/deployment/bind/feature_spec/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deployment/bind/feature_spec/output.txt b/acceptance/bundle/deployment/bind/feature_spec/output.txt new file mode 100644 index 00000000000..ed1a48d5540 --- /dev/null +++ b/acceptance/bundle/deployment/bind/feature_spec/output.txt @@ -0,0 +1,53 @@ + +>>> [CLI] schemas create fspec_[UNIQUE_NAME] main -o json +{ + "full_name": "main.fspec_[UNIQUE_NAME]" +} + +>>> create_feature_spec_fixtures.py main.fspec_[UNIQUE_NAME] + +=== Create the feature spec outside the bundle +>>> [CLI] api post /api/2.0/feature-engineering/feature-specs --json { + "name": "main.fspec_[UNIQUE_NAME].boundspec", + "features": [{"feature_lookup": {"table": "main.fspec_[UNIQUE_NAME].user_features", "lookup_keys": ["user_id"], "feature_names": ["age"]}}] +} +{ + "name": "main.fspec_[UNIQUE_NAME].boundspec" +} + +>>> [CLI] bundle deployment bind spec1 main.fspec_[UNIQUE_NAME].boundspec --auto-approve +Successfully bound feature_spec with an id 'main.fspec_[UNIQUE_NAME].boundspec' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle summary +Name: test-bind-feature-spec-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bind-feature-spec-[UNIQUE_NAME]/default +Resources: + Feature Specs: + spec1: + Name: main.fspec_[UNIQUE_NAME].boundspec + URL: [DATABRICKS_URL]/explore/data/functions/main/fspec_[UNIQUE_NAME]/boundspec?w=[WORKSPACE_ID] + +=== The bound feature spec matches the config, so there is nothing to change +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deployment unbind spec1 + +>>> [CLI] bundle summary +Name: test-bind-feature-spec-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bind-feature-spec-[UNIQUE_NAME]/default +Resources: + Feature Specs: + spec1: + Name: main.fspec_[UNIQUE_NAME].boundspec + URL: (not deployed) + +>>> [CLI] api delete /api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].boundspec +{} diff --git a/acceptance/bundle/deployment/bind/feature_spec/script b/acceptance/bundle/deployment/bind/feature_spec/script new file mode 100644 index 00000000000..cbe4e43b07e --- /dev/null +++ b/acceptance/bundle/deployment/bind/feature_spec/script @@ -0,0 +1,29 @@ +# Creating a feature spec resolves its lookup tables in Unity Catalog, so the test creates them in +# a schema of its own. Dropping the schema also drops the feature spec. +export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME" +trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}' +trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" + +cleanup() { + $CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml + +title "Create the feature spec outside the bundle" +MSYS_NO_PATHCONV=1 trace $CLI api post /api/2.0/feature-engineering/feature-specs --json "{ + \"name\": \"$SCHEMA_FULL_NAME.boundspec\", + \"features\": [{\"feature_lookup\": {\"table\": \"$SCHEMA_FULL_NAME.user_features\", \"lookup_keys\": [\"user_id\"], \"feature_names\": [\"age\"]}}] +}" | jq '{name}' + +trace $CLI bundle deployment bind spec1 $SCHEMA_FULL_NAME.boundspec --auto-approve +trace $CLI bundle summary + +title "The bound feature spec matches the config, so there is nothing to change" +trace $CLI bundle plan + +trace $CLI bundle deployment unbind spec1 +trace $CLI bundle summary + +MSYS_NO_PATHCONV=1 trace $CLI api delete /api/2.0/feature-engineering/feature-specs/$SCHEMA_FULL_NAME.boundspec diff --git a/acceptance/bundle/deployment/bind/feature_spec/test.toml b/acceptance/bundle/deployment/bind/feature_spec/test.toml new file mode 100644 index 00000000000..9f8601174e5 --- /dev/null +++ b/acceptance/bundle/deployment/bind/feature_spec/test.toml @@ -0,0 +1,8 @@ +Ignore = [ + ".databricks", +] + +# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' diff --git a/acceptance/bundle/invariant/auto-migrate/test.toml b/acceptance/bundle/invariant/auto-migrate/test.toml index d930af8ee12..ee0a30ed9da 100644 --- a/acceptance/bundle/invariant/auto-migrate/test.toml +++ b/acceptance/bundle/invariant/auto-migrate/test.toml @@ -43,6 +43,8 @@ EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"] EnvMatrixExclude.no_data_quality_monitor = ["INPUT_CONFIG=data_quality_monitor.yml.tmpl"] # Features are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_feature = ["INPUT_CONFIG=feature.yml.tmpl"] +# Feature specs are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_feature_spec = ["INPUT_CONFIG=feature_spec.yml.tmpl"] # The Postgres Data API is direct-only; the terraform deploy that seeds the migration fails for it. EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"] diff --git a/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl new file mode 100644 index 00000000000..876cc6dc22e --- /dev/null +++ b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + feature_specs: + foo: + name: main.fspec_inv_$UNIQUE_NAME.test_feature_spec + features: + - feature_lookup: + table: main.fspec_inv_$UNIQUE_NAME.user_features + lookup_keys: [user_id] + feature_names: [age] + grants: + - principal: account users + privileges: + - EXECUTE diff --git a/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-cleanup.sh b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-cleanup.sh new file mode 100755 index 00000000000..94d24f624b3 --- /dev/null +++ b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-cleanup.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "Cleaning up temporary schema" +$CLI schemas delete main.fspec_inv_$UNIQUE_NAME --force || true diff --git a/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh new file mode 100755 index 00000000000..91158f1d7ce --- /dev/null +++ b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Creating a feature spec resolves its lookup table in Unity Catalog, and the invariant harness +# provisions no tables of its own. +echo "Creating temporary schema and feature table: main.fspec_inv_$UNIQUE_NAME" + +$CLI schemas create fspec_inv_$UNIQUE_NAME main -o json | jq '{full_name}' +create_feature_spec_fixtures.py main.fspec_inv_$UNIQUE_NAME diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 8f82295fb33..86ddfee7460 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -17,6 +17,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "feature.yml.tmpl", + "feature_spec.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_run_job_ref.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 25bbc5b91f4..9de4b5dba9b 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "feature.yml.tmpl", + "feature_spec.yml.tmpl", "genie_space.yml.tmpl", "instance_pool.yml.tmpl", "job.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 25bbc5b91f4..9de4b5dba9b 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "feature.yml.tmpl", + "feature_spec.yml.tmpl", "genie_space.yml.tmpl", "instance_pool.yml.tmpl", "job.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 87defc2a4ce..71e5463fa01 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -43,6 +43,8 @@ EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"] EnvMatrixExclude.no_data_quality_monitor = ["INPUT_CONFIG=data_quality_monitor.yml.tmpl"] # Features are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_feature = ["INPUT_CONFIG=feature.yml.tmpl"] +# Feature specs are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_feature_spec = ["INPUT_CONFIG=feature_spec.yml.tmpl"] # The Postgres Data API is direct-only; the terraform deploy that seeds the migration fails for it. EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 750656d62f7..a2856e6353d 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "feature.yml.tmpl", + "feature_spec.yml.tmpl", "genie_space.yml.tmpl", "instance_pool.yml.tmpl", "job.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index dd6d5c33e3f..04a46bf44d4 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -33,6 +33,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "feature.yml.tmpl", + "feature_spec.yml.tmpl", "genie_space.yml.tmpl", "instance_pool.yml.tmpl", "job.yml.tmpl", @@ -125,6 +126,8 @@ EnvMatrixExclude.no_data_quality_monitor_on_cloud = ["CONFIG_Cloud=true", "INPUT # Deployment history has no "features." resource-key prefix, so the deploy fails before the feature # is created; see dmsResourceTypes in libs/testserver/bundledeployments.go. EnvMatrixExclude.no_feature_dms = ["DMS=true", "INPUT_CONFIG=feature.yml.tmpl"] +# Same for "feature_specs.". +EnvMatrixExclude.no_feature_spec_dms = ["DMS=true", "INPUT_CONFIG=feature_spec.yml.tmpl"] # Same for the "postgres_data_apis." prefix. EnvMatrixExclude.no_postgres_data_api_dms = ["DMS=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"] # Fake SQL endpoint for local tests diff --git a/acceptance/bundle/python/feature_specs-support/databricks.yml b/acceptance/bundle/python/feature_specs-support/databricks.yml new file mode 100644 index 00000000000..c4ded89a6eb --- /dev/null +++ b/acceptance/bundle/python/feature_specs-support/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: my_project + +sync: {paths: []} # don't need to copy files + +python: + resources: + - "resources:load_resources" + mutators: + - "mutators:update_feature_spec" + +resources: + feature_specs: + my_feature_spec_1: + name: main.default.my_feature_spec_1 + features: + - feature_lookup: + table: main.default.user_features + lookup_keys: [user_id] + feature_names: [age] + grants: + - principal: account users + privileges: [EXECUTE] diff --git a/acceptance/bundle/python/feature_specs-support/mutators.py b/acceptance/bundle/python/feature_specs-support/mutators.py new file mode 100644 index 00000000000..3a7ac22e95c --- /dev/null +++ b/acceptance/bundle/python/feature_specs-support/mutators.py @@ -0,0 +1,11 @@ +from dataclasses import replace + +from databricks.bundles.core import feature_spec_mutator +from databricks.bundles.feature_specs import FeatureSpec + + +@feature_spec_mutator +def update_feature_spec(feature_spec: FeatureSpec) -> FeatureSpec: + assert isinstance(feature_spec.name, str) + + return replace(feature_spec, name=f"{feature_spec.name}_updated") diff --git a/acceptance/bundle/python/feature_specs-support/out.test.toml b/acceptance/bundle/python/feature_specs-support/out.test.toml new file mode 100644 index 00000000000..ea77b758035 --- /dev/null +++ b/acceptance/bundle/python/feature_specs-support/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] +EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/feature_specs-support/output.txt b/acceptance/bundle/python/feature_specs-support/output.txt new file mode 100644 index 00000000000..db824f305b3 --- /dev/null +++ b/acceptance/bundle/python/feature_specs-support/output.txt @@ -0,0 +1,57 @@ + +>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json +{ + "experimental": { + "python": { + "mutators": [ + "mutators:update_feature_spec" + ], + "resources": [ + "resources:load_resources" + ] + } + }, + "resources": { + "feature_specs": { + "my_feature_spec_1": { + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.default.user_features" + } + } + ], + "grants": [ + { + "principal": "account users", + "privileges": [ + "EXECUTE" + ] + } + ], + "name": "main.default.my_feature_spec_1_updated" + }, + "my_feature_spec_2": { + "extra_pip_requirements": { + "requirements": [ + "numpy==2.1.0" + ] + }, + "features": [ + { + "declarative_feature": { + "feature": "main.default.my_feature" + } + } + ], + "name": "main.default.my_feature_spec_2_updated" + } + } + } +} diff --git a/acceptance/bundle/python/feature_specs-support/resources.py b/acceptance/bundle/python/feature_specs-support/resources.py new file mode 100644 index 00000000000..9aa289ab86f --- /dev/null +++ b/acceptance/bundle/python/feature_specs-support/resources.py @@ -0,0 +1,18 @@ +from databricks.bundles.core import Resources + + +def load_resources() -> Resources: + resources = Resources() + + resources.add_feature_spec( + "my_feature_spec_2", + { + "name": "main.default.my_feature_spec_2", + "features": [ + {"declarative_feature": {"feature": "main.default.my_feature"}}, + ], + "extra_pip_requirements": {"requirements": ["numpy==2.1.0"]}, + }, + ) + + return resources diff --git a/acceptance/bundle/python/feature_specs-support/script b/acceptance/bundle/python/feature_specs-support/script new file mode 100644 index 00000000000..dde070e89da --- /dev/null +++ b/acceptance/bundle/python/feature_specs-support/script @@ -0,0 +1,4 @@ +trace uv run $UV_ARGS -q $CLI bundle validate --output json | \ + jq "pick(.experimental.python, .resources)" + +rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/python/feature_specs-support/test.toml b/acceptance/bundle/python/feature_specs-support/test.toml new file mode 100644 index 00000000000..6aee90e30c0 --- /dev/null +++ b/acceptance/bundle/python/feature_specs-support/test.toml @@ -0,0 +1,4 @@ +Cloud = true + +# feature specs are only supported in the current version of the wheel +EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/refschema/output/feature_specs.txt b/acceptance/bundle/refschema/output/feature_specs.txt new file mode 100644 index 00000000000..da911bcda5c --- /dev/null +++ b/acceptance/bundle/refschema/output/feature_specs.txt @@ -0,0 +1,48 @@ +resources.feature_specs.*.exclude_columns []string ALL +resources.feature_specs.*.exclude_columns[*] string ALL +resources.feature_specs.*.extra_pip_requirements *ml.PipRequirements ALL +resources.feature_specs.*.extra_pip_requirements.requirements []string ALL +resources.feature_specs.*.extra_pip_requirements.requirements[*] string ALL +resources.feature_specs.*.features []ml.Features ALL +resources.feature_specs.*.features[*] ml.Features ALL +resources.feature_specs.*.features[*].declarative_feature *ml.DeclarativeFeature ALL +resources.feature_specs.*.features[*].declarative_feature.feature string ALL +resources.feature_specs.*.features[*].feature_function *ml.FeatureFunction ALL +resources.feature_specs.*.features[*].feature_function.input_bindings []ml.FeatureFunctionInputBinding ALL +resources.feature_specs.*.features[*].feature_function.input_bindings[*] ml.FeatureFunctionInputBinding ALL +resources.feature_specs.*.features[*].feature_function.input_bindings[*].bound_feature string ALL +resources.feature_specs.*.features[*].feature_function.input_bindings[*].parameter string ALL +resources.feature_specs.*.features[*].feature_function.output string ALL +resources.feature_specs.*.features[*].feature_function.udf string ALL +resources.feature_specs.*.features[*].feature_lookup *ml.FeatureLookup ALL +resources.feature_specs.*.features[*].feature_lookup.default_values []ml.DefaultValue ALL +resources.feature_specs.*.features[*].feature_lookup.default_values[*] ml.DefaultValue ALL +resources.feature_specs.*.features[*].feature_lookup.default_values[*].default_value string ALL +resources.feature_specs.*.features[*].feature_lookup.default_values[*].feature string ALL +resources.feature_specs.*.features[*].feature_lookup.feature_names []string ALL +resources.feature_specs.*.features[*].feature_lookup.feature_names[*] string ALL +resources.feature_specs.*.features[*].feature_lookup.lookback_window *duration.Duration ALL +resources.feature_specs.*.features[*].feature_lookup.lookup_keys []string ALL +resources.feature_specs.*.features[*].feature_lookup.lookup_keys[*] string ALL +resources.feature_specs.*.features[*].feature_lookup.rename_outputs []ml.RenameOutput ALL +resources.feature_specs.*.features[*].feature_lookup.rename_outputs[*] ml.RenameOutput ALL +resources.feature_specs.*.features[*].feature_lookup.rename_outputs[*].feature string ALL +resources.feature_specs.*.features[*].feature_lookup.rename_outputs[*].output string ALL +resources.feature_specs.*.features[*].feature_lookup.table string ALL +resources.feature_specs.*.features[*].feature_lookup.timestamp_lookup_keys []string ALL +resources.feature_specs.*.features[*].feature_lookup.timestamp_lookup_keys[*] string ALL +resources.feature_specs.*.id string INPUT +resources.feature_specs.*.lifecycle resources.Lifecycle INPUT +resources.feature_specs.*.lifecycle.depends_on []resources.DependsOn INPUT +resources.feature_specs.*.lifecycle.depends_on[*] resources.DependsOn INPUT +resources.feature_specs.*.lifecycle.depends_on[*].resource string INPUT +resources.feature_specs.*.lifecycle.prevent_destroy bool INPUT +resources.feature_specs.*.modified_status string INPUT +resources.feature_specs.*.name string ALL +resources.feature_specs.*.url string INPUT +resources.feature_specs.*.grants.full_name string ALL +resources.feature_specs.*.grants.securable_type string ALL +resources.feature_specs.*.grants[*] catalog.PrivilegeAssignment ALL +resources.feature_specs.*.grants[*].principal string ALL +resources.feature_specs.*.grants[*].privileges []catalog.Privilege ALL +resources.feature_specs.*.grants[*].privileges[*] catalog.Privilege ALL diff --git a/acceptance/bundle/resource_deps/implicit_deps_feature_spec/databricks.yml b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/databricks.yml new file mode 100644 index 00000000000..74553f37d67 --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/databricks.yml @@ -0,0 +1,25 @@ +bundle: + name: test-bundle + +# Tests implicit dependency detection for feature specs: +# - the catalog and schema parts of name should resolve +resources: + catalogs: + my_catalog: + name: mycatalog + schemas: + my_schema: + catalog_name: mycatalog + name: myschema + feature_specs: + my_feature_spec: + name: mycatalog.myschema.myspec + features: + - feature_lookup: + table: mycatalog.myschema.user_features + lookup_keys: [user_id] + feature_names: [age] + +targets: + dev: + mode: development diff --git a/acceptance/bundle/resource_deps/implicit_deps_feature_spec/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_feature_spec/output.txt b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/output.txt new file mode 100644 index 00000000000..f1663050f87 --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle validate -t dev -o json +{ + "catalogs": { + "my_catalog": { + "name": "mycatalog" + } + }, + "feature_specs": { + "my_feature_spec": { + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age" + ], + "lookup_keys": [ + "user_id" + ], + "table": "mycatalog.myschema.user_features" + } + } + ], + "name": "${resources.catalogs.my_catalog.name}.${resources.schemas.my_schema.name}.myspec" + } + }, + "schemas": { + "my_schema": { + "catalog_name": "${resources.catalogs.my_catalog.name}", + "name": "dev_[USERNAME]_myschema" + } + } +} diff --git a/acceptance/bundle/resource_deps/implicit_deps_feature_spec/script b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/script new file mode 100644 index 00000000000..30cb3ec2e5d --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/script @@ -0,0 +1 @@ +trace $CLI bundle validate -t dev -o json | jq .resources diff --git a/acceptance/bundle/resource_deps/implicit_deps_feature_spec/test.toml b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_feature_spec/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/bundle/resources/feature_specs/basic/databricks.yml.tmpl b/acceptance/bundle/resources/feature_specs/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..f9894902a32 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/basic/databricks.yml.tmpl @@ -0,0 +1,20 @@ +bundle: + name: test-feature-specs-basic-$UNIQUE_NAME + +resources: + feature_specs: + spec1: + name: $SCHEMA_FULL_NAME.myspec + features: + - feature_lookup: + table: $SCHEMA_FULL_NAME.user_features + lookup_keys: [user_id] + feature_names: [age, country] + - feature_function: + udf: $SCHEMA_FULL_NAME.score + input_bindings: + - parameter: x + bound_feature: age + output: score + extra_pip_requirements: + requirements: ["numpy==2.1.0"] diff --git a/acceptance/bundle/resources/feature_specs/basic/out.test.toml b/acceptance/bundle/resources/feature_specs/basic/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/basic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/feature_specs/basic/output.txt b/acceptance/bundle/resources/feature_specs/basic/output.txt new file mode 100644 index 00000000000..985a38a245b --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/basic/output.txt @@ -0,0 +1,181 @@ + +>>> [CLI] schemas create fspec_[UNIQUE_NAME] main -o json +{ + "full_name": "main.fspec_[UNIQUE_NAME]" +} + +>>> create_feature_spec_fixtures.py main.fspec_[UNIQUE_NAME] + +=== Initial summary before deploy +>>> [CLI] bundle summary -o json +{ + "extra_pip_requirements": { + "requirements": [ + "numpy==2.1.0" + ] + }, + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age", + "country" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.fspec_[UNIQUE_NAME].user_features" + } + }, + { + "feature_function": { + "input_bindings": [ + { + "bound_feature": "age", + "parameter": "x" + } + ], + "output": "score", + "udf": "main.fspec_[UNIQUE_NAME].score" + } + } + ], + "modified_status": "created", + "name": "[SPEC1_ID]" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-basic-[UNIQUE_NAME]/default/files... +Created feature_specs.spec1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //feature-engineering +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/feature-specs", + "body": { + "extra_pip_requirements": { + "requirements": [ + "numpy==2.1.0" + ] + }, + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age", + "country" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.fspec_[UNIQUE_NAME].user_features" + } + }, + { + "feature_function": { + "input_bindings": [ + { + "bound_feature": "age", + "parameter": "x" + } + ], + "output": "score", + "udf": "main.fspec_[UNIQUE_NAME].score" + } + } + ], + "name": "[SPEC1_ID]" + } +} + +=== Summary should show the id and the Catalog Explorer url +>>> [CLI] bundle summary -o json +{ + "id": "[SPEC1_ID]", + "url": "[DATABRICKS_URL]/explore/data/functions/main/fspec_[UNIQUE_NAME]/myspec?w=[WORKSPACE_ID]" +} + +=== A second plan on an unchanged bundle must report no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Change a field (every field is immutable, so this recreates) +>>> update_file.py databricks.yml [age, country] [age] + +>>> [CLI] bundle plan +recreate feature_specs.spec1 + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-basic-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following feature specs. +Grants on them that are not declared in the bundle are lost: + recreate resources.feature_specs.spec1 +Recreated feature_specs.spec1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> print_requests.py //feature-engineering +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/feature-specs/[SPEC1_ID]" +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/feature-specs", + "body": { + "extra_pip_requirements": { + "requirements": [ + "numpy==2.1.0" + ] + }, + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.fspec_[UNIQUE_NAME].user_features" + } + }, + { + "feature_function": { + "input_bindings": [ + { + "bound_feature": "age", + "parameter": "x" + } + ], + "output": "score", + "udf": "main.fspec_[UNIQUE_NAME].score" + } + } + ], + "name": "[SPEC1_ID]" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.feature_specs.spec1 + +This action will result in the deletion of the following feature specs. +Feature Serving endpoints and training code that use them can no longer load them: + delete resources.feature_specs.spec1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-basic-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py //feature-engineering +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/feature-specs/[SPEC1_ID]" +} diff --git a/acceptance/bundle/resources/feature_specs/basic/script b/acceptance/bundle/resources/feature_specs/basic/script new file mode 100644 index 00000000000..b61b43d2bc3 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/basic/script @@ -0,0 +1,23 @@ +feature_spec_setup + +title "Initial summary before deploy" +trace $CLI bundle summary -o json | jq .resources.feature_specs.spec1 + +trace $CLI bundle deploy +replace_ids.py +trace print_requests.py //feature-engineering + +title "Summary should show the id and the Catalog Explorer url" +trace $CLI bundle summary -o json | jq ".resources.feature_specs.spec1 | {id, url}" + +title "A second plan on an unchanged bundle must report no changes" +trace $CLI bundle plan + +title "Change a field (every field is immutable, so this recreates)" +trace update_file.py databricks.yml "[age, country]" "[age]" +trace $CLI bundle plan +trace $CLI bundle deploy --auto-approve +trace print_requests.py //feature-engineering + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //feature-engineering diff --git a/acceptance/bundle/resources/feature_specs/basic/test.toml b/acceptance/bundle/resources/feature_specs/basic/test.toml new file mode 100644 index 00000000000..bd4b66fe75e --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/basic/test.toml @@ -0,0 +1,3 @@ +Ignore = [ + ".databricks", +] diff --git a/acceptance/bundle/resources/feature_specs/order/databricks.yml.tmpl b/acceptance/bundle/resources/feature_specs/order/databricks.yml.tmpl new file mode 100644 index 00000000000..dfb2f5b8ef7 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/order/databricks.yml.tmpl @@ -0,0 +1,27 @@ +bundle: + name: test-feature-specs-order-$UNIQUE_NAME + +resources: + feature_specs: + spec1: + name: $SCHEMA_FULL_NAME.myspec + features: + - feature_lookup: + table: $SCHEMA_FULL_NAME.tenant_user_features + lookup_keys: [tenant_id, user_id] + feature_names: [age, country] + rename_outputs: + - {feature: age, output: user_age} + - {feature: country, output: user_country} + default_values: + - {feature: user_age, default_value: "0"} + - {feature: user_country, default_value: US} + - feature_function: + udf: $SCHEMA_FULL_NAME.country_score + input_bindings: + - {parameter: x, bound_feature: user_age} + - {parameter: y, bound_feature: user_country} + output: score + exclude_columns: [user_age, user_country] + extra_pip_requirements: + requirements: [numpy, scipy] diff --git a/acceptance/bundle/resources/feature_specs/order/out.test.toml b/acceptance/bundle/resources/feature_specs/order/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/order/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/feature_specs/order/output.txt b/acceptance/bundle/resources/feature_specs/order/output.txt new file mode 100644 index 00000000000..ea08d118636 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/order/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] schemas create fspec_[UNIQUE_NAME] main -o json +{ + "full_name": "main.fspec_[UNIQUE_NAME]" +} + +>>> create_feature_spec_fixtures.py main.fspec_[UNIQUE_NAME] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-order-[UNIQUE_NAME]/default/files... +Created feature_specs.spec1 +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Reorder the lists whose order means nothing: no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Also reorder the lookup keys, which must follow the table's primary key: recreate +>>> [CLI] bundle plan +recreate feature_specs.spec1 + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.feature_specs.spec1 + +This action will result in the deletion of the following feature specs. +Feature Serving endpoints and training code that use them can no longer load them: + delete resources.feature_specs.spec1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-order-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/feature_specs/order/reordered-lookup-keys.yml.tmpl b/acceptance/bundle/resources/feature_specs/order/reordered-lookup-keys.yml.tmpl new file mode 100644 index 00000000000..9b193865e5d --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/order/reordered-lookup-keys.yml.tmpl @@ -0,0 +1,27 @@ +bundle: + name: test-feature-specs-order-$UNIQUE_NAME + +resources: + feature_specs: + spec1: + name: $SCHEMA_FULL_NAME.myspec + features: + - feature_lookup: + table: $SCHEMA_FULL_NAME.tenant_user_features + lookup_keys: [user_id, tenant_id] + feature_names: [age, country] + rename_outputs: + - {feature: country, output: user_country} + - {feature: age, output: user_age} + default_values: + - {feature: user_country, default_value: US} + - {feature: user_age, default_value: "0"} + - feature_function: + udf: $SCHEMA_FULL_NAME.country_score + input_bindings: + - {parameter: y, bound_feature: user_country} + - {parameter: x, bound_feature: user_age} + output: score + exclude_columns: [user_country, user_age] + extra_pip_requirements: + requirements: [scipy, numpy] diff --git a/acceptance/bundle/resources/feature_specs/order/reordered.yml.tmpl b/acceptance/bundle/resources/feature_specs/order/reordered.yml.tmpl new file mode 100644 index 00000000000..a8025062359 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/order/reordered.yml.tmpl @@ -0,0 +1,27 @@ +bundle: + name: test-feature-specs-order-$UNIQUE_NAME + +resources: + feature_specs: + spec1: + name: $SCHEMA_FULL_NAME.myspec + features: + - feature_lookup: + table: $SCHEMA_FULL_NAME.tenant_user_features + lookup_keys: [tenant_id, user_id] + feature_names: [age, country] + rename_outputs: + - {feature: country, output: user_country} + - {feature: age, output: user_age} + default_values: + - {feature: user_country, default_value: US} + - {feature: user_age, default_value: "0"} + - feature_function: + udf: $SCHEMA_FULL_NAME.country_score + input_bindings: + - {parameter: y, bound_feature: user_country} + - {parameter: x, bound_feature: user_age} + output: score + exclude_columns: [user_country, user_age] + extra_pip_requirements: + requirements: [scipy, numpy] diff --git a/acceptance/bundle/resources/feature_specs/order/script b/acceptance/bundle/resources/feature_specs/order/script new file mode 100644 index 00000000000..c2a7e04ae7b --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/order/script @@ -0,0 +1,13 @@ +feature_spec_setup + +trace $CLI bundle deploy + +title "Reorder the lists whose order means nothing: no changes" +envsubst < reordered.yml.tmpl > databricks.yml +trace $CLI bundle plan + +title "Also reorder the lookup keys, which must follow the table's primary key: recreate" +envsubst < reordered-lookup-keys.yml.tmpl > databricks.yml +trace $CLI bundle plan + +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/feature_specs/order/test.toml b/acceptance/bundle/resources/feature_specs/order/test.toml new file mode 100644 index 00000000000..daa475f7cee --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/order/test.toml @@ -0,0 +1,5 @@ +RecordRequests = false + +Ignore = [ + ".databricks", +] diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/databricks.yml.tmpl b/acceptance/bundle/resources/feature_specs/remote-delete/databricks.yml.tmpl new file mode 100644 index 00000000000..300b003c587 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/remote-delete/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: test-feature-specs-remote-delete-$UNIQUE_NAME + +resources: + feature_specs: + spec1: + name: $SCHEMA_FULL_NAME.myspec + features: + - feature_lookup: + table: $SCHEMA_FULL_NAME.user_features + lookup_keys: [user_id] + feature_names: [age] diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/out.test.toml b/acceptance/bundle/resources/feature_specs/remote-delete/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/remote-delete/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/output.txt b/acceptance/bundle/resources/feature_specs/remote-delete/output.txt new file mode 100644 index 00000000000..4fe99398c4b --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/remote-delete/output.txt @@ -0,0 +1,28 @@ + +>>> [CLI] schemas create fspec_[UNIQUE_NAME] main -o json +{ + "full_name": "main.fspec_[UNIQUE_NAME]" +} + +>>> create_feature_spec_fixtures.py main.fspec_[UNIQUE_NAME] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-remote-delete-[UNIQUE_NAME]/default/files... +Created feature_specs.spec1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Delete the feature spec out of band +>>> [CLI] api delete /api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].myspec +{} + +=== Plan should detect the feature spec is gone and re-create it +>>> [CLI] bundle plan +create feature_specs.spec1 + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-remote-delete-[UNIQUE_NAME]/default + +Destroy: 0 deleted diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/script b/acceptance/bundle/resources/feature_specs/remote-delete/script new file mode 100644 index 00000000000..a0142c4c72d --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/remote-delete/script @@ -0,0 +1,11 @@ +feature_spec_setup + +trace $CLI bundle deploy + +title "Delete the feature spec out of band" +MSYS_NO_PATHCONV=1 trace $CLI api delete /api/2.0/feature-engineering/feature-specs/$SCHEMA_FULL_NAME.myspec + +title "Plan should detect the feature spec is gone and re-create it" +trace $CLI bundle plan + +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/test.toml b/acceptance/bundle/resources/feature_specs/remote-delete/test.toml new file mode 100644 index 00000000000..daa475f7cee --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/remote-delete/test.toml @@ -0,0 +1,5 @@ +RecordRequests = false + +Ignore = [ + ".databricks", +] diff --git a/acceptance/bundle/resources/feature_specs/script.prepare b/acceptance/bundle/resources/feature_specs/script.prepare new file mode 100644 index 00000000000..0529e3ea8d2 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/script.prepare @@ -0,0 +1,15 @@ +# Creating a feature spec resolves its lookup tables and UDFs in Unity Catalog, so each test creates +# them in a schema of its own. Dropping the schema also drops any feature spec a failed run leaves. +feature_spec_cleanup() { + $CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1 || true + $CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true +} + +# Call from the test's script, so the trap belongs to the shell that runs it. +feature_spec_setup() { + export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME" + trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}' + trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" + trap feature_spec_cleanup EXIT + envsubst < databricks.yml.tmpl > databricks.yml +} diff --git a/acceptance/bundle/resources/feature_specs/test.toml b/acceptance/bundle/resources/feature_specs/test.toml new file mode 100644 index 00000000000..eebb76502b2 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/test.toml @@ -0,0 +1,11 @@ +Cloud = true + +# Deployment history has no "feature_specs." resource-key prefix, so the deploy fails before the +# feature spec is created. See dmsResourceTypes in libs/testserver/bundledeployments.go, which +# mirrors Universe. +EnvMatrixExclude.no_feature_spec_dms = ["DMS=true"] + +# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' diff --git a/acceptance/bundle/resources/feature_specs/uppercase-name/databricks.yml.tmpl b/acceptance/bundle/resources/feature_specs/uppercase-name/databricks.yml.tmpl new file mode 100644 index 00000000000..2e179632982 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/uppercase-name/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: test-feature-specs-uppercase-name-$UNIQUE_NAME + +resources: + feature_specs: + spec1: + name: $SCHEMA_FULL_NAME.MySpec + features: + - feature_lookup: + table: $SCHEMA_FULL_NAME.user_features + lookup_keys: [user_id] + feature_names: [age] diff --git a/acceptance/bundle/resources/feature_specs/uppercase-name/out.test.toml b/acceptance/bundle/resources/feature_specs/uppercase-name/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/uppercase-name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt b/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt new file mode 100644 index 00000000000..67caf37b353 --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt @@ -0,0 +1,118 @@ + +>>> [CLI] schemas create fspec_[UNIQUE_NAME] main -o json +{ + "full_name": "main.fspec_[UNIQUE_NAME]" +} + +>>> create_feature_spec_fixtures.py main.fspec_[UNIQUE_NAME] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-uppercase-name-[UNIQUE_NAME]/default/files... +Created feature_specs.spec1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //feature-engineering +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/feature-specs", + "body": { + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.fspec_[UNIQUE_NAME].user_features" + } + } + ], + "name": "main.fspec_[UNIQUE_NAME].MySpec" + } +} + +=== UC lowercases the name, so the id is lowercase +>>> [CLI] bundle summary -o json +"main.fspec_[UNIQUE_NAME].myspec" + +=== A plan on an unchanged bundle skips the name instead of recreating the feature spec +>>> [CLI] bundle plan -o json +{ + "resources.feature_specs.spec1": { + "action": "skip", + "changes": { + "name": { + "action": "skip", + "reason": "id_field" + } + } + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Renaming the feature spec still recreates it +>>> update_file.py databricks.yml .MySpec .MyOtherSpec + +>>> [CLI] bundle plan +recreate feature_specs.spec1 + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-uppercase-name-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following feature specs. +Grants on them that are not declared in the bundle are lost: + recreate resources.feature_specs.spec1 +Recreated feature_specs.spec1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> print_requests.py //feature-engineering +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].myspec" +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/feature-specs", + "body": { + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.fspec_[UNIQUE_NAME].user_features" + } + } + ], + "name": "main.fspec_[UNIQUE_NAME].MyOtherSpec" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.feature_specs.spec1 + +This action will result in the deletion of the following feature specs. +Feature Serving endpoints and training code that use them can no longer load them: + delete resources.feature_specs.spec1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-uppercase-name-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py //feature-engineering +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].myotherspec" +} diff --git a/acceptance/bundle/resources/feature_specs/uppercase-name/script b/acceptance/bundle/resources/feature_specs/uppercase-name/script new file mode 100644 index 00000000000..bd515213f2f --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/uppercase-name/script @@ -0,0 +1,21 @@ +feature_spec_setup + +# No replace_ids.py, so the output shows the id in lowercase. +trace $CLI bundle deploy +trace print_requests.py //feature-engineering + +title "UC lowercases the name, so the id is lowercase" +trace $CLI bundle summary -o json | jq .resources.feature_specs.spec1.id + +title "A plan on an unchanged bundle skips the name instead of recreating the feature spec" +trace $CLI bundle plan -o json | jq '.plan | map_values({action, changes: (.changes // {} | map_values({action, reason}))})' +trace $CLI bundle plan + +title "Renaming the feature spec still recreates it" +trace update_file.py databricks.yml ".MySpec" ".MyOtherSpec" +trace $CLI bundle plan +trace $CLI bundle deploy --auto-approve +trace print_requests.py //feature-engineering + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //feature-engineering diff --git a/acceptance/bundle/resources/feature_specs/uppercase-name/test.toml b/acceptance/bundle/resources/feature_specs/uppercase-name/test.toml new file mode 100644 index 00000000000..bd4b66fe75e --- /dev/null +++ b/acceptance/bundle/resources/feature_specs/uppercase-name/test.toml @@ -0,0 +1,3 @@ +Ignore = [ + ".databricks", +] diff --git a/acceptance/bundle/resources/grants/feature_specs/databricks.yml.tmpl b/acceptance/bundle/resources/grants/feature_specs/databricks.yml.tmpl new file mode 100644 index 00000000000..38f420fb004 --- /dev/null +++ b/acceptance/bundle/resources/grants/feature_specs/databricks.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: deploy-feature-spec-grants-$UNIQUE_NAME + +resources: + feature_specs: + my_feature_spec: + name: $SCHEMA_FULL_NAME.myspec + features: + - feature_lookup: + table: $SCHEMA_FULL_NAME.user_features + lookup_keys: [user_id] + feature_names: [age, country] + grants: + - principal: account users + privileges: ["EXECUTE"] diff --git a/acceptance/bundle/resources/grants/feature_specs/out.deploy1.requests.json b/acceptance/bundle/resources/grants/feature_specs/out.deploy1.requests.json new file mode 100644 index 00000000000..3992bb98363 --- /dev/null +++ b/acceptance/bundle/resources/grants/feature_specs/out.deploy1.requests.json @@ -0,0 +1,17 @@ +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/permissions/function/main.fspec_[UNIQUE_NAME].myspec", + "body": { + "changes": [ + { + "add": [ + "EXECUTE" + ], + "principal": "account users", + "remove": [ + "ALL_PRIVILEGES" + ] + } + ] + } +} diff --git a/acceptance/bundle/resources/grants/feature_specs/out.deploy2.requests.json b/acceptance/bundle/resources/grants/feature_specs/out.deploy2.requests.json new file mode 100644 index 00000000000..eb1cff5a806 --- /dev/null +++ b/acceptance/bundle/resources/grants/feature_specs/out.deploy2.requests.json @@ -0,0 +1,41 @@ +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].myspec" +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/feature-specs", + "body": { + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.fspec_[UNIQUE_NAME].user_features" + } + } + ], + "name": "main.fspec_[UNIQUE_NAME].myspec" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/permissions/function/main.fspec_[UNIQUE_NAME].myspec", + "body": { + "changes": [ + { + "add": [ + "EXECUTE" + ], + "principal": "account users", + "remove": [ + "ALL_PRIVILEGES" + ] + } + ] + } +} diff --git a/acceptance/bundle/resources/grants/feature_specs/out.destroy.requests.json b/acceptance/bundle/resources/grants/feature_specs/out.destroy.requests.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/grants/feature_specs/out.plan1.json b/acceptance/bundle/resources/grants/feature_specs/out.plan1.json new file mode 100644 index 00000000000..67401dddfbb --- /dev/null +++ b/acceptance/bundle/resources/grants/feature_specs/out.plan1.json @@ -0,0 +1,54 @@ +{ + "plan_version": [PLAN_VERSION], + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.feature_specs.my_feature_spec": { + "action": "create", + "new_state": { + "value": { + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age", + "country" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.fspec_[UNIQUE_NAME].user_features" + } + } + ], + "name": "main.fspec_[UNIQUE_NAME].myspec" + } + } + }, + "resources.feature_specs.my_feature_spec.grants": { + "depends_on": [ + { + "node": "resources.feature_specs.my_feature_spec", + "label": "${resources.feature_specs.my_feature_spec.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "securable_type": "function", + "full_name": "", + "__embed__": [ + { + "principal": "account users", + "privileges": [ + "EXECUTE" + ] + } + ] + }, + "vars": { + "full_name": "${resources.feature_specs.my_feature_spec.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resources/grants/feature_specs/out.plan2.json b/acceptance/bundle/resources/grants/feature_specs/out.plan2.json new file mode 100644 index 00000000000..cb953c10535 --- /dev/null +++ b/acceptance/bundle/resources/grants/feature_specs/out.plan2.json @@ -0,0 +1,110 @@ +{ + "plan_version": [PLAN_VERSION], + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.feature_specs.my_feature_spec": { + "action": "recreate", + "new_state": { + "value": { + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.fspec_[UNIQUE_NAME].user_features" + } + } + ], + "name": "main.fspec_[UNIQUE_NAME].myspec" + } + }, + "remote_state": { + "features": [ + { + "feature_lookup": { + "feature_names": [ + "age", + "country" + ], + "lookup_keys": [ + "user_id" + ], + "table": "main.fspec_[UNIQUE_NAME].user_features" + } + } + ], + "name": "main.fspec_[UNIQUE_NAME].myspec" + }, + "changes": { + "features[0].feature_lookup.feature_names": { + "action": "recreate", + "reason": "spec:immutable", + "old": [ + "age", + "country" + ], + "new": [ + "age" + ], + "remote": [ + "age", + "country" + ] + } + } + }, + "resources.feature_specs.my_feature_spec.grants": { + "depends_on": [ + { + "node": "resources.feature_specs.my_feature_spec", + "label": "${resources.feature_specs.my_feature_spec.id}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "securable_type": "function", + "full_name": "", + "__embed__": [ + { + "principal": "account users", + "privileges": [ + "EXECUTE" + ] + } + ] + }, + "vars": { + "full_name": "${resources.feature_specs.my_feature_spec.id}" + } + }, + "remote_state": { + "securable_type": "function", + "full_name": "main.fspec_[UNIQUE_NAME].myspec", + "__embed__": [ + { + "principal": "account users", + "privileges": [ + "EXECUTE" + ] + } + ] + }, + "changes": { + "full_name": { + "action": "update_id", + "reason": "id_changes", + "old": "main.fspec_[UNIQUE_NAME].myspec", + "new": "", + "remote": "main.fspec_[UNIQUE_NAME].myspec" + } + } + } + } +} diff --git a/acceptance/bundle/resources/grants/feature_specs/out.test.toml b/acceptance/bundle/resources/grants/feature_specs/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/grants/feature_specs/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/grants/feature_specs/output.txt b/acceptance/bundle/resources/grants/feature_specs/output.txt new file mode 100644 index 00000000000..3a9978724f2 --- /dev/null +++ b/acceptance/bundle/resources/grants/feature_specs/output.txt @@ -0,0 +1,76 @@ + +>>> [CLI] schemas create fspec_[UNIQUE_NAME] main -o json +{ + "full_name": "main.fspec_[UNIQUE_NAME]" +} + +>>> create_feature_spec_fixtures.py main.fspec_[UNIQUE_NAME] + +>>> [CLI] bundle plan -o json + +>>> print_requests.py --get //permissions + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-feature-spec-grants-[UNIQUE_NAME]/default/files... +Created feature_specs.my_feature_spec +Created feature_specs.my_feature_spec.grants +Files: 3 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //permissions + +>>> [CLI] grants get function main.fspec_[UNIQUE_NAME].myspec +{ + "privilege_assignments": [ + { + "principal": "account users", + "privileges": [ + "EXECUTE" + ] + } + ] +} + +=== Every change recreates the feature spec, which drops its grants; the bundle must re-apply them +>>> update_file.py databricks.yml [age, country] [age] + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-feature-spec-grants-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following feature specs. +Grants on them that are not declared in the bundle are lost: + recreate resources.feature_specs.my_feature_spec +Recreated feature_specs.my_feature_spec +Updated feature_specs.my_feature_spec.grants +Files: 3 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 0 unchanged + +>>> print_requests.py //feature-engineering //permissions + +>>> [CLI] grants get function main.fspec_[UNIQUE_NAME].myspec +{ + "privilege_assignments": [ + { + "principal": "account users", + "privileges": [ + "EXECUTE" + ] + } + ] +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.feature_specs.my_feature_spec + +This action will result in the deletion of the following feature specs. +Feature Serving endpoints and training code that use them can no longer load them: + delete resources.feature_specs.my_feature_spec + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-feature-spec-grants-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py //permissions diff --git a/acceptance/bundle/resources/grants/feature_specs/script b/acceptance/bundle/resources/grants/feature_specs/script new file mode 100644 index 00000000000..bc39ede6df8 --- /dev/null +++ b/acceptance/bundle/resources/grants/feature_specs/script @@ -0,0 +1,29 @@ +# Creating a feature spec resolves its lookup tables in Unity Catalog, so the test creates them in +# a schema of its own. +export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME" +trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}' +trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" + +cleanup() { + $CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1 || true + $CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml + +trace $CLI bundle plan -o json | jq 'del(.features)' > out.plan1.json +trace print_requests.py --get //permissions +trace $CLI bundle deploy +trace print_requests.py //permissions > out.deploy1.requests.json +trace $CLI grants get function $SCHEMA_FULL_NAME.myspec | jq --sort-keys + +title "Every change recreates the feature spec, which drops its grants; the bundle must re-apply them" +trace update_file.py databricks.yml "[age, country]" "[age]" +trace $CLI bundle plan -o json | jq 'del(.features)' > out.plan2.json +trace $CLI bundle deploy --auto-approve +trace print_requests.py //feature-engineering //permissions > out.deploy2.requests.json +trace $CLI grants get function $SCHEMA_FULL_NAME.myspec | jq --sort-keys + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //permissions > out.destroy.requests.json diff --git a/acceptance/bundle/resources/grants/feature_specs/test.toml b/acceptance/bundle/resources/grants/feature_specs/test.toml new file mode 100644 index 00000000000..08fbace3299 --- /dev/null +++ b/acceptance/bundle/resources/grants/feature_specs/test.toml @@ -0,0 +1,8 @@ +# Deployment history has no "feature_specs." resource-key prefix; see dmsResourceTypes in +# libs/testserver/bundledeployments.go. +EnvMatrixExclude.no_feature_spec_dms = ["DMS=true"] + +# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index f178342ef54..f976b1b1794 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, features, genie_spaces, instance_pools, jobs, mcp_services, model_provider_services, model_services, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, feature_specs, features, genie_spaces, instance_pools, jobs, mcp_services, model_provider_services, model_services, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler >>> [CLI] __complete experimental open , @@ -22,6 +22,7 @@ dashboards database_catalogs database_instances experiments +feature_specs features genie_spaces instance_pools diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 00eff169286..a88a6f37a3e 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -26,6 +26,7 @@ var unsupportedResources = []string{ "schemas", "features", "materialization_sets", + "feature_specs", "quality_monitors", "data_quality_monitors", "registered_models", diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 4953d5df25e..30a2d8d427c 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -181,6 +181,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }}, }, + FeatureSpecs: map[string]*resources.FeatureSpec{ + "feature_spec1": {FeatureSpec: ml.FeatureSpec{Name: "catalog1.schema1.feature_spec1"}}, + }, Clusters: map[string]*resources.Cluster{ "cluster1": {ClusterSpec: compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}}, }, @@ -525,6 +528,7 @@ func TestProcessTargetModeDefault(t *testing.T) { assert.Equal(t, "schema1", b.Config.Resources.Schemas["schema1"].Name) assert.Equal(t, "volume1", b.Config.Resources.Volumes["volume1"].Name) assert.Equal(t, "catalog1.schema1.feature1", b.Config.Resources.Features["feature1"].FullName) + assert.Equal(t, "catalog1.schema1.feature_spec1", b.Config.Resources.FeatureSpecs["feature_spec1"].Name) assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) assert.Equal(t, "instance_pool1", b.Config.Resources.InstancePools["instance_pool1"].InstancePoolName) assert.Equal(t, "sql_warehouse1", b.Config.Resources.SqlWarehouses["sql_warehouse1"].Name) @@ -570,6 +574,7 @@ func TestAppropriateResourcesAreRenamed(t *testing.T) { notUserNamed := []string{ "Apps", "Features", + "FeatureSpecs", "SecretScopes", "Secrets", "DatabaseInstances", diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index 0b1b91dabe6..dd54bae891e 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -218,6 +218,20 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia } } } + for _, fs := range b.Config.Resources.FeatureSpecs { + if fs == nil { + continue + } + if dynvar.ContainsVariableReference(fs.Name) { + continue + } + parts, ok := splitUCName(fs.Name, 3) + if !ok { + continue + } + catalogName, schemaName := parts[0], parts[1] + fs.Name = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] + } for _, mse := range b.Config.Resources.ModelServingEndpoints { if mse == nil { continue diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index ab89caf9914..6979407c4ec 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -49,6 +49,7 @@ func allResourceTypes(t *testing.T) []string { "database_instances", "experiments", "external_locations", + "feature_specs", "features", "genie_spaces", "instance_pools", @@ -208,6 +209,7 @@ var allowList = []string{ "postgres_synced_tables", "registered_models", "experiments", + "feature_specs", "features", "genie_spaces", "instance_pools", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index a937cb73a76..aba758405e1 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -18,6 +18,7 @@ type Resources struct { Features map[string]*resources.Feature `json:"features,omitempty"` MaterializationSets map[string]*resources.MaterializationSet `json:"materialization_sets,omitempty"` + FeatureSpecs map[string]*resources.FeatureSpec `json:"feature_specs,omitempty"` Models map[string]*resources.MlflowModel `json:"models,omitempty"` Experiments map[string]*resources.MlflowExperiment `json:"experiments,omitempty"` ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"` @@ -120,6 +121,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["experiments"], r.Experiments), collectResourceMap(descriptions["features"], r.Features), collectResourceMap(descriptions["materialization_sets"], r.MaterializationSets), + collectResourceMap(descriptions["feature_specs"], r.FeatureSpecs), collectResourceMap(descriptions["model_serving_endpoints"], r.ModelServingEndpoints), collectResourceMap(descriptions["model_services"], r.ModelServices), collectResourceMap(descriptions["mcp_services"], r.McpServices), @@ -210,6 +212,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), "features": (&resources.Feature{}).ResourceDescription(), "materialization_sets": (&resources.MaterializationSet{}).ResourceDescription(), + "feature_specs": (&resources.FeatureSpec{}).ResourceDescription(), "instance_pools": (&resources.InstancePool{}).ResourceDescription(), "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), "model_services": (&resources.ModelService{}).ResourceDescription(), diff --git a/bundle/config/resources/feature_spec.go b/bundle/config/resources/feature_spec.go new file mode 100644 index 00000000000..9472d6826f8 --- /dev/null +++ b/bundle/config/resources/feature_spec.go @@ -0,0 +1,69 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/databricks-sdk-go/apierr" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +type FeatureSpec struct { + BaseResource + ID string `json:"id,omitempty" bundle:"readonly"` + ml.FeatureSpec + + // List of grants to apply on this feature spec. + Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` +} + +func (f *FeatureSpec) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, f) +} + +func (f FeatureSpec) MarshalJSON() ([]byte, error) { + return marshal.Marshal(f) +} + +func (f *FeatureSpec) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.FeatureEngineering.GetFeatureSpec(ctx, ml.GetFeatureSpecRequest{ + Name: name, + }) + if err != nil { + log.Debugf(ctx, "feature spec with name %s does not exist: %v", name, err) + + if apierr.IsMissing(err) { + return false, nil + } + + return false, err + } + + return true, nil +} + +func (*FeatureSpec) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "feature_spec", + PluralName: "feature_specs", + SingularTitle: "Feature Spec", + PluralTitle: "Feature Specs", + } +} + +func (f *FeatureSpec) InitializeURL(baseURL url.URL) { + if f.ID == "" { + return + } + f.URL = workspaceurls.ResourceURL(baseURL, "feature_specs", f.ID) +} + +func (f *FeatureSpec) GetName() string { + return f.Name +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index d01cd27efcd..9e20d4139e7 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -390,6 +390,13 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + FeatureSpecs: map[string]*resources.FeatureSpec{ + "my_feature_spec": { + FeatureSpec: ml.FeatureSpec{ + Name: "main.myschema.my_feature_spec", + }, + }, + }, PostgresDataApis: map[string]*resources.PostgresDataApi{ "my_postgres_data_api": { PostgresDataApiConfig: resources.PostgresDataApiConfig{ @@ -429,6 +436,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockExperimentsAPI().EXPECT().GetExperiment(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockFeatureEngineeringAPI().EXPECT().GetFeature(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockFeatureEngineeringAPI().EXPECT().GetMaterializationSet(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockFeatureEngineeringAPI().EXPECT().GetFeatureSpec(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockRegisteredModelsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockCatalogsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockExternalLocationsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 1b1d2f15ebf..0bc31f6642d 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -13,6 +13,7 @@ var SupportedResources = map[string]any{ "experiments": (*ResourceExperiment)(nil), "features": (*ResourceFeature)(nil), "materialization_sets": (*ResourceMaterializationSet)(nil), + "feature_specs": (*ResourceFeatureSpec)(nil), "catalogs": (*ResourceCatalog)(nil), "schemas": (*ResourceSchema)(nil), "external_locations": (*ResourceExternalLocation)(nil), @@ -75,6 +76,7 @@ var SupportedResources = map[string]any{ "external_locations.grants": (*ResourceGrants)(nil), "volumes.grants": (*ResourceGrants)(nil), "registered_models.grants": (*ResourceGrants)(nil), + "feature_specs.grants": (*ResourceGrants)(nil), "vector_search_indexes.grants": (*ResourceGrants)(nil), "secrets.grants": (*ResourceGrants)(nil), "model_services.grants": (*ResourceGrants)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index b542481be83..605f530a94c 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -150,6 +150,18 @@ var testConfig map[string]any = map[string]any{ }, }, + "feature_specs": &resources.FeatureSpec{ + Name: "main.default.my_feature_spec", + Features: []ml.Features{{ + FeatureLookup: &ml.FeatureLookup{ + Table: "main.default.my_table", + LookupKeys: []string{"id"}, + FeatureNames: []string{"amount"}, + }, + }}, + ExtraPipRequirements: &ml.PipRequirements{Requirements: []string{"numpy==2.1.0"}}, + }, + "experiments": &resources.MlflowExperiment{ Name: "my-experiment", Tags: []ml.ExperimentTag{ @@ -740,6 +752,17 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "feature_specs.grants": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + return &GrantsState{ + SecurableType: "function", + FullName: "main.default.my_feature_spec", + EmbeddedSlice: []catalog.PrivilegeAssignment{{ + Privileges: []catalog.Privilege{catalog.PrivilegeExecute}, + Principal: "user@example.com", + }}, + }, nil + }, + "vector_search_indexes.grants": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { return &GrantsState{ SecurableType: "table", diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index d3a230b347d..956cb8c30a7 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -22,6 +22,8 @@ experiments: ml.CreateExperiment external_locations: catalog.CreateExternalLocation +feature_specs: ml.FeatureSpec + features: ml.Feature genie_spaces: dashboards.GenieUpdateSpaceRequest diff --git a/bundle/direct/dresources/configs/feature_specs.generated.yml b/bundle/direct/dresources/configs/feature_specs.generated.yml new file mode 100644 index 00000000000..ebdc4dd6e56 --- /dev/null +++ b/bundle/direct/dresources/configs/feature_specs.generated.yml @@ -0,0 +1,11 @@ +# Generated, do not edit. + +recreate_on_changes: + - field: exclude_columns + reason: spec:immutable + - field: extra_pip_requirements + reason: spec:immutable + - field: features + reason: spec:immutable + - field: name + reason: spec:immutable diff --git a/bundle/direct/dresources/configs/feature_specs.yml b/bundle/direct/dresources/configs/feature_specs.yml new file mode 100644 index 00000000000..48edeb6e434 --- /dev/null +++ b/bundle/direct/dresources/configs/feature_specs.yml @@ -0,0 +1,4 @@ +provided_id_fields: + # UC lowercases the name; remote returns "main.ml.myspec" for config "main.ml.MySpec". + - field: name + reason: id_field diff --git a/bundle/direct/dresources/feature_spec.go b/bundle/direct/dresources/feature_spec.go new file mode 100644 index 00000000000..078e26e3b4b --- /dev/null +++ b/bundle/direct/dresources/feature_spec.go @@ -0,0 +1,95 @@ +package dresources + +import ( + "context" + "slices" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// https://docs.databricks.com/api/workspace/featureengineering +// The API has no update: every field is immutable, so any change recreates the feature spec. +type ResourceFeatureSpec struct { + client *databricks.WorkspaceClient +} + +func (*ResourceFeatureSpec) New(client *databricks.WorkspaceClient) *ResourceFeatureSpec { + return &ResourceFeatureSpec{client: client} +} + +// PrepareState sorts the fields in FeatureSpec that are order-free lists, so a reorder in config is +// not a recreate. The FeatureSpec with the order-free list fields sorted is what gets sent and +// stored in the backend. +func (*ResourceFeatureSpec) PrepareState(input *resources.FeatureSpec) *ml.FeatureSpec { + state := input.FeatureSpec + sortOrderFreeLists(&state) + return &state +} + +func (r *ResourceFeatureSpec) DoRead(ctx context.Context, id string) (*ml.FeatureSpec, error) { + featureSpec, err := r.client.FeatureEngineering.GetFeatureSpec(ctx, ml.GetFeatureSpecRequest{Name: id}) + if err != nil { + return nil, err + } + // Returns the FeatureSpec with its order-free list fields sorted, so a feature spec created + // outside the bundle also has them sorted. + sortOrderFreeLists(featureSpec) + return featureSpec, nil +} + +func (r *ResourceFeatureSpec) DoCreate(ctx context.Context, config *ml.FeatureSpec) (string, *ml.FeatureSpec, error) { + response, err := r.client.FeatureEngineering.CreateFeatureSpec(ctx, ml.CreateFeatureSpecRequest{FeatureSpec: *config}) + if err != nil { + return "", nil, err + } + return response.Name, response, nil +} + +func (r *ResourceFeatureSpec) DoDelete(ctx context.Context, id string, _ *ml.FeatureSpec) error { + return r.client.FeatureEngineering.DeleteFeatureSpec(ctx, ml.DeleteFeatureSpecRequest{Name: id}) +} + +// KeyedSlices compares the fields in FeatureSpec that consist of a list of key-value pairs by key +// instead of position, since their order means nothing. The other list fields in FeatureSpec are +// still compared positionally. +func (*ResourceFeatureSpec) KeyedSlices() map[string]any { + return map[string]any{ + "features[*].feature_lookup.rename_outputs": renameOutputKey, + "features[*].feature_lookup.default_values": defaultValueKey, + "features[*].feature_function.input_bindings": featureFunctionInputBindingKey, + } +} + +func renameOutputKey(x ml.RenameOutput) (string, string) { + return "feature", x.Feature +} + +func defaultValueKey(x ml.DefaultValue) (string, string) { + return "feature", x.Feature +} + +func featureFunctionInputBindingKey(x ml.FeatureFunctionInputBinding) (string, string) { + return "parameter", x.Parameter +} + +// sortOrderFreeLists sorts a FeatureSpec's ExcludeColumns and ExtraPipRequirements. Both are +// treated as sets by the server but stored as string lists, which cannot be compared by a key +// defined in KeyedSlices. Each field is replaced with the sorted list rather than sorted in place, +// so the bundle config is not mutated. +func sortOrderFreeLists(featureSpec *ml.FeatureSpec) { + featureSpec.ExcludeColumns = sortedClone(featureSpec.ExcludeColumns) + if featureSpec.ExtraPipRequirements != nil { + pipRequirements := *featureSpec.ExtraPipRequirements + pipRequirements.Requirements = sortedClone(pipRequirements.Requirements) + featureSpec.ExtraPipRequirements = &pipRequirements + } +} + +// sortedClone keeps an empty list empty rather than nil, so the request body is unchanged. +func sortedClone(values []string) []string { + sorted := slices.Clone(values) + slices.Sort(sorted) + return sorted +} diff --git a/bundle/direct/dresources/feature_spec_test.go b/bundle/direct/dresources/feature_spec_test.go new file mode 100644 index 00000000000..2788d371b92 --- /dev/null +++ b/bundle/direct/dresources/feature_spec_test.go @@ -0,0 +1,126 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFeatureSpecPrepareStateSortsOrderFreeListsWithoutMutatingConfig(t *testing.T) { + input := &resources.FeatureSpec{ + Name: "main.default.spec", + ExcludeColumns: []string{"b", "a"}, + ExtraPipRequirements: &ml.PipRequirements{Requirements: []string{"scipy", "numpy"}}, + } + + state := (&ResourceFeatureSpec{}).PrepareState(input) + + assert.Equal(t, []string{"a", "b"}, state.ExcludeColumns) + assert.Equal(t, []string{"numpy", "scipy"}, state.ExtraPipRequirements.Requirements) + assert.Equal(t, []string{"b", "a"}, input.ExcludeColumns) + assert.Equal(t, []string{"scipy", "numpy"}, input.ExtraPipRequirements.Requirements) +} + +func TestFeatureSpecPrepareStateKeepsEmptyListsEmpty(t *testing.T) { + input := &resources.FeatureSpec{ + Name: "main.default.spec", + ExtraPipRequirements: &ml.PipRequirements{Requirements: []string{}}, + } + + state := (&ResourceFeatureSpec{}).PrepareState(input) + + assert.Nil(t, state.ExcludeColumns) + assert.NotNil(t, state.ExtraPipRequirements.Requirements) + assert.Empty(t, state.ExtraPipRequirements.Requirements) +} + +func TestFeatureSpecDoReadSortsOrderFreeLists(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceFeatureSpec{}).New(m.WorkspaceClient) + + m.GetMockFeatureEngineeringAPI().EXPECT(). + GetFeatureSpec(ctx, ml.GetFeatureSpecRequest{Name: "main.default.spec"}). + Return(&ml.FeatureSpec{ + Name: "main.default.spec", + ExcludeColumns: []string{"b", "a"}, + ExtraPipRequirements: &ml.PipRequirements{Requirements: []string{"scipy", "numpy"}}, + }, nil). + Once() + + remote, err := r.DoRead(ctx, "main.default.spec") + require.NoError(t, err) + + assert.Equal(t, []string{"a", "b"}, remote.ExcludeColumns) + assert.Equal(t, []string{"numpy", "scipy"}, remote.ExtraPipRequirements.Requirements) +} + +func featureSpecWithLookupAndFunction(keys []string, renames []ml.RenameOutput, defaults []ml.DefaultValue, bindings []ml.FeatureFunctionInputBinding) *ml.FeatureSpec { + return &ml.FeatureSpec{ + Name: "main.default.spec", + Features: []ml.Features{ + {FeatureLookup: &ml.FeatureLookup{ + Table: "main.default.users", + LookupKeys: keys, + FeatureNames: []string{"age", "country"}, + RenameOutputs: renames, + DefaultValues: defaults, + }}, + {FeatureFunction: &ml.FeatureFunction{ + Udf: "main.default.score", + InputBindings: bindings, + Output: "score", + }}, + }, + } +} + +func TestFeatureSpecKeyedSlicesIgnoreOrderOfMapLikeLists(t *testing.T) { + keys := (&ResourceFeatureSpec{}).KeyedSlices() + + config := featureSpecWithLookupAndFunction( + []string{"user_id"}, + []ml.RenameOutput{{Feature: "age", Output: "user_age"}, {Feature: "country", Output: "user_country"}}, + []ml.DefaultValue{{Feature: "user_age", DefaultValue: "0"}, {Feature: "user_country", DefaultValue: "US"}}, + []ml.FeatureFunctionInputBinding{{Parameter: "x", BoundFeature: "user_age"}, {Parameter: "y", BoundFeature: "user_country"}}, + ) + remote := featureSpecWithLookupAndFunction( + []string{"user_id"}, + []ml.RenameOutput{{Feature: "country", Output: "user_country"}, {Feature: "age", Output: "user_age"}}, + []ml.DefaultValue{{Feature: "user_country", DefaultValue: "US"}, {Feature: "user_age", DefaultValue: "0"}}, + []ml.FeatureFunctionInputBinding{{Parameter: "y", BoundFeature: "user_country"}, {Parameter: "x", BoundFeature: "user_age"}}, + ) + + changes, err := structdiff.GetStructDiff(config, remote, keys) + require.NoError(t, err) + assert.Empty(t, changes) + + // A real edit inside a keyed list is still a change. + remote.Features[0].FeatureLookup.RenameOutputs[0].Output = "renamed_country" + changes, err = structdiff.GetStructDiff(config, remote, keys) + require.NoError(t, err) + assert.NotEmpty(t, changes) +} + +func TestFeatureSpecKeyedSlicesKeepPositionalListsOrdered(t *testing.T) { + keys := (&ResourceFeatureSpec{}).KeyedSlices() + + // Lookup keys must follow the feature table's primary key order. + config := featureSpecWithLookupAndFunction([]string{"a", "b"}, nil, nil, nil) + remote := featureSpecWithLookupAndFunction([]string{"b", "a"}, nil, nil, nil) + changes, err := structdiff.GetStructDiff(config, remote, keys) + require.NoError(t, err) + assert.NotEmpty(t, changes) + + // The order of features sets the output column order. + remote = featureSpecWithLookupAndFunction([]string{"a", "b"}, nil, nil, nil) + remote.Features[0], remote.Features[1] = remote.Features[1], remote.Features[0] + changes, err = structdiff.GetStructDiff(config, remote, keys) + require.NoError(t, err) + assert.NotEmpty(t, changes) +} diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 378149b9a2f..005305804e5 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -18,6 +18,7 @@ var grantResourceToSecurableType = map[string]string{ "external_locations": "external_location", "volumes": "volume", "registered_models": "function", + "feature_specs": "function", "secrets": "secret", "vector_search_indexes": "table", "model_services": "model_service", diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 08b65053e3b..861d2241277 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -952,6 +952,32 @@ resources: "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "feature_specs": + "description": |- + The feature spec definitions for the bundle, where each key is the name of the feature spec. + "markdown_description": |- + The feature spec definitions for the bundle, where each key is the name of the feature spec. + "$fields": + "grants": + "description": |- + The Unity Catalog privileges to grant to principals on this securable. + "markdown_description": |- + A Sequence of Unity Catalog privilege grants on this securable, where each item grants a set of `privileges` to a `principal` (a user, group, or service principal). + + See [\_](/data-governance/unity-catalog/manage-privileges/index.md). + "markdown_examples": |- + ```yaml + grants: + - principal: account users + privileges: + - EXECUTE + - principal: data-engineers + privileges: + - ALL_PRIVILEGES + ``` + "lifecycle": + "description": |- + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. "features": "description": |- The feature definitions for the bundle, where each key is the name of the feature. diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 6b2ad1f369d..760d14d9178 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -93,6 +93,16 @@ var RequiredFields = map[string][]string{ "resources.external_locations.*": {"credential_name", "name", "url"}, "resources.external_locations.*.lifecycle.depends_on[*]": {"resource"}, + "resources.feature_specs.*": {"name"}, + "resources.feature_specs.*.extra_pip_requirements": {"requirements"}, + "resources.feature_specs.*.features[*].declarative_feature": {"feature"}, + "resources.feature_specs.*.features[*].feature_function": {"udf"}, + "resources.feature_specs.*.features[*].feature_function.input_bindings[*]": {"bound_feature", "parameter"}, + "resources.feature_specs.*.features[*].feature_lookup": {"lookup_keys", "table"}, + "resources.feature_specs.*.features[*].feature_lookup.default_values[*]": {"feature"}, + "resources.feature_specs.*.features[*].feature_lookup.rename_outputs[*]": {"feature", "output"}, + "resources.feature_specs.*.lifecycle.depends_on[*]": {"resource"}, + "resources.features.*": {"full_name", "function", "source"}, "resources.features.*.entities[*]": {"name"}, "resources.features.*.function.aggregation_function.approx_count_distinct": {"input"}, diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index c3c3306130e..9c4d4b178cb 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -42,6 +42,7 @@ var deployApprovalGroups = []approvalGroup{ {group: "genie_spaces", message: deleteOrRecreateGenieSpaceMessage}, {group: "materialization_sets", message: deleteOrRecreateMaterializationSetMessage}, {group: "features", message: deleteOrRecreateFeatureMessage}, + {group: "feature_specs", message: deleteOrRecreateFeatureSpecMessage}, } func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (bool, error) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index b3e885e54d4..07fed65ba80 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -46,6 +46,7 @@ var destroyApprovalGroups = []approvalGroup{ {group: "vector_search_indexes", message: deleteVectorSearchIndexMessage}, {group: "genie_spaces", message: deleteGenieSpaceMessage}, {group: "features", message: deleteFeatureMessage}, + {group: "feature_specs", message: deleteFeatureSpecMessage}, } // logPipelineDeleteApproval prints the pipeline deletions. If cascade_on_destroy is true, we will include diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index cad9d6f7bf0..0e7a29a9c82 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -61,6 +61,10 @@ A Feature View that has materialized features in a materialization set cannot be recreated. Remove its materialized features from the set in a separate deploy first, then delete or recreate the Feature View. Deleting or recreating a Feature View that a FeatureSpec serves is not blocked, and leaves the FeatureSpec stale:` + + deleteOrRecreateFeatureSpecMessage = ` +This action will result in the deletion or recreation of the following feature specs. +Grants on them that are not declared in the bundle are lost:` ) // DataLossWarning is the warning shown when a non-interactive command is about @@ -114,4 +118,7 @@ The conversation history attached to them is permanently lost:` deleteFeatureMessage = `This action will result in the deletion of the following Feature Views. Deleting a Feature View that a FeatureSpec serves is not blocked, and leaves the FeatureSpec stale:` + + deleteFeatureSpecMessage = `This action will result in the deletion of the following feature specs. +Feature Serving endpoints and training code that use them can no longer load them:` ) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2bcd9701fce..5476a32ea3f 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1176,6 +1176,57 @@ } ] }, + "resources.FeatureSpec": { + "oneOf": [ + { + "type": "object", + "description": "A named set of features to serve, stored as a Unity Catalog function.", + "properties": { + "exclude_columns": { + "description": "[Private Preview] Output names of columns to leave out of the feature spec.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "extra_pip_requirements": { + "description": "[Private Preview] Extra pip requirements to install in the serving container, in addition to the defaults and\nthose resolved from the features.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.PipRequirements", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "features": { + "description": "[Private Preview] The Features (either a declarative one, or a lookup from a feature table, or a feature\nfunction) to serve. A declarative feature (`DeclarativeFeature`) cannot be combined with\nlookups (`FeatureLookup`) and functions (`FeatureFunction`) in one feature spec.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.Features", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "grants": { + "description": "The Unity Catalog privileges to grant to principals on this securable.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment", + "markdownDescription": "A Sequence of Unity Catalog privilege grants on this securable, where each item grants a set of `privileges` to a `principal` (a user, group, or service principal).\n\nSee [link](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/index.html)." + }, + "lifecycle": { + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "name": { + "description": "[Private Preview] Three-part name (catalog.schema.feature_spec) of the feature spec.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.GenieSpace": { "oneOf": [ { @@ -4356,6 +4407,11 @@ "external_locations": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation" }, + "feature_specs": { + "description": "The feature spec definitions for the bundle, where each key is the name of the feature spec.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.FeatureSpec", + "markdownDescription": "The feature spec definitions for the bundle, where each key is the name of the feature spec." + }, "features": { "description": "The feature definitions for the bundle, where each key is the name of the feature.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Feature", @@ -14053,6 +14109,60 @@ } ] }, + "ml.DeclarativeFeature": { + "oneOf": [ + { + "type": "object", + "description": "A reference to a Feature.", + "properties": { + "feature": { + "description": "[Private Preview] Three-part Unity Catalog name (catalog.schema.feature) of the Feature.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "feature" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.DefaultValue": { + "oneOf": [ + { + "type": "object", + "description": "A default value for a looked-up feature.", + "properties": { + "default_value": { + "description": "[Private Preview] Default value for the feature.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "feature": { + "description": "[Private Preview] Name of the feature: its output name if it is renamed, otherwise its name in the feature table.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "feature" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.DeltaTableSource": { "oneOf": [ { @@ -14191,6 +14301,134 @@ } ] }, + "ml.FeatureFunction": { + "oneOf": [ + { + "type": "object", + "description": "A Python UDF that computes a feature.", + "properties": { + "input_bindings": { + "description": "[Private Preview] Bindings of the UDF's parameters to features.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.FeatureFunctionInputBinding", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "output": { + "description": "[Private Preview] Output name of the feature that the UDF computes.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "udf": { + "description": "[Private Preview] Three-part name of the Python UDF.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "udf" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.FeatureFunctionInputBinding": { + "oneOf": [ + { + "type": "object", + "description": "Binds one parameter of the UDF to a feature.", + "properties": { + "bound_feature": { + "description": "[Private Preview] Name of the feature bound to the parameter.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "parameter": { + "description": "[Private Preview] Name of the UDF parameter.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "bound_feature", + "parameter" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.FeatureLookup": { + "oneOf": [ + { + "type": "object", + "description": "Features to look up from a feature table.", + "properties": { + "default_values": { + "description": "[Private Preview] Default values for features.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.DefaultValue", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "feature_names": { + "description": "[Private Preview] Names of the features to look up, or empty to look up every feature in the table.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "lookback_window": { + "description": "[Private Preview] Lookback window to use when performing point-in-time lookup.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/common/types/duration.Duration", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "lookup_keys": { + "description": "[Private Preview] Keys to use when joining with the feature table.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "rename_outputs": { + "description": "[Private Preview] Output names for features, in place of their names in the feature table.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.RenameOutput", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "table": { + "description": "[Private Preview] Three-part name of the feature table.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "timestamp_lookup_keys": { + "description": "[Private Preview] Keys to use when performing point-in-time lookup.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "lookup_keys", + "table" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.FeatureReference": { "oneOf": [ { @@ -14236,6 +14474,39 @@ } ] }, + "ml.Features": { + "oneOf": [ + { + "type": "object", + "description": "One entry of a feature spec: a Feature, a feature table lookup, or a feature function.", + "properties": { + "declarative_feature": { + "description": "[Private Preview] A Feature, referenced by its three-part Unity Catalog name.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.DeclarativeFeature", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "feature_function": { + "description": "[Private Preview] A feature function.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.FeatureFunction", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "feature_lookup": { + "description": "[Private Preview] A feature table lookup.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.FeatureLookup", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.FieldDefinition": { "oneOf": [ { @@ -15040,6 +15311,30 @@ } ] }, + "ml.PipRequirements": { + "oneOf": [ + { + "type": "object", + "description": "Pip requirements to install in a serving container.", + "properties": { + "requirements": { + "description": "[Private Preview] Pip requirement strings (PEP 508) or UC Volume whl paths (/Volumes/...).", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "requirements" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.RegisteredModelPermissionLevel": { "oneOf": [ { @@ -15059,6 +15354,37 @@ } ] }, + "ml.RenameOutput": { + "oneOf": [ + { + "type": "object", + "description": "An output name for a looked-up feature.", + "properties": { + "feature": { + "description": "[Private Preview] Name of the feature in the feature table.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "output": { + "description": "[Private Preview] Output name for the feature.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "feature", + "output" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.RequestSource": { "oneOf": [ { @@ -21708,6 +22034,20 @@ } ] }, + "resources.FeatureSpec": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.FeatureSpec" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.GenieSpace": { "oneOf": [ { @@ -23027,6 +23367,20 @@ } ] }, + "ml.DefaultValue": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.DefaultValue" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.EntityColumn": { "oneOf": [ { @@ -23055,6 +23409,20 @@ } ] }, + "ml.FeatureFunctionInputBinding": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.FeatureFunctionInputBinding" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.FeatureReference": { "oneOf": [ { @@ -23069,6 +23437,20 @@ } ] }, + "ml.Features": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.Features" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.FieldDefinition": { "oneOf": [ { @@ -23139,6 +23521,20 @@ } ] }, + "ml.RenameOutput": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.RenameOutput" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.DayOfWeek": { "oneOf": [ { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 9634c01a287..ac1c272d013 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -34,6 +34,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.experiments.test_mlflow_experiment": {ID: "1"}, "resources.features.test_feature": {ID: "main.default.test_feature"}, "resources.materialization_sets.test_materialization_set": {ID: "set1"}, + "resources.feature_specs.test_feature_spec": {ID: "main.default.test_feature_spec"}, "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, "resources.model_services.test_model_service": {ID: "main.default.test_model_service"}, "resources.data_quality_monitors.test_data_quality_monitor": {ID: "table,uuid-1"}, @@ -244,6 +245,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + FeatureSpecs: map[string]*resources.FeatureSpec{ + "test_feature_spec": { + FeatureSpec: ml.FeatureSpec{ + Name: "main.default.test_feature_spec", + }, + }, + }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "test_model_serving": { CreateServingEndpoint: serving.CreateServingEndpoint{ @@ -710,6 +718,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + FeatureSpecs: map[string]*resources.FeatureSpec{ + "test_feature_spec": { + FeatureSpec: ml.FeatureSpec{ + Name: "main.default.test_feature_spec", + }, + }, + "test_feature_spec_new": { + FeatureSpec: ml.FeatureSpec{ + Name: "main.default.test_feature_spec_new", + }, + }, + }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "test_model_serving": { CreateServingEndpoint: serving.CreateServingEndpoint{ @@ -1142,6 +1162,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.experiments.test_mlflow_experiment_old": {ID: "2"}, "resources.features.test_feature": {ID: "main.default.test_feature"}, "resources.features.test_feature_old": {ID: "main.default.test_feature_old"}, + "resources.feature_specs.test_feature_spec": {ID: "main.default.test_feature_spec"}, + "resources.feature_specs.test_feature_spec_old": {ID: "main.default.test_feature_spec_old"}, "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, "resources.model_serving_endpoints.test_model_serving_old": {ID: "2"}, "resources.registered_models.test_registered_model": {ID: "1"}, @@ -1253,6 +1275,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.Features["test_feature_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Features["test_feature_new"].ModifiedStatus) + assert.Equal(t, "main.default.test_feature_spec", config.Resources.FeatureSpecs["test_feature_spec"].ID) + assert.Empty(t, config.Resources.FeatureSpecs["test_feature_spec"].ModifiedStatus) + assert.Equal(t, "main.default.test_feature_spec_old", config.Resources.FeatureSpecs["test_feature_spec_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.FeatureSpecs["test_feature_spec_old"].ModifiedStatus) + assert.Empty(t, config.Resources.FeatureSpecs["test_feature_spec_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.FeatureSpecs["test_feature_spec_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.ModelServingEndpoints["test_model_serving"].ID) assert.Empty(t, config.Resources.ModelServingEndpoints["test_model_serving"].ModifiedStatus) assert.Equal(t, "2", config.Resources.ModelServingEndpoints["test_model_serving_old"].ID) diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 91aa6d07f63..dd62f40177a 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -35,6 +35,7 @@ func TestBuildWorkspaceURLPathBasedResources(t *testing.T) { {"clusters", "0123-456789-abc", "https://myworkspace.databricks.com/compute/clusters/0123-456789-abc"}, {"registered_models", "catalog.schema.model", "https://myworkspace.databricks.com/explore/data/models/catalog/schema/model"}, {"features", "catalog.schema.feature", "https://myworkspace.databricks.com/explore/data/features/catalog/schema/feature"}, + {"feature_specs", "catalog.schema.spec", "https://myworkspace.databricks.com/explore/data/functions/catalog/schema/spec"}, } for _, tt := range tests { @@ -68,7 +69,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, features, genie_spaces, instance_pools, jobs, mcp_services, model_provider_services, model_services, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, feature_specs, features, genie_spaces, instance_pools, jobs, mcp_services, model_provider_services, model_services, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -117,6 +118,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "database_catalogs", "database_instances", "experiments", + "feature_specs", "features", "genie_spaces", "instance_pools", @@ -154,7 +156,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, features, genie_spaces, instance_pools, jobs, mcp_services, model_provider_services, model_services, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, feature_specs, features, genie_spaces, instance_pools, jobs, mcp_services, model_provider_services, model_services, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index c48ebb04df2..98c52d1fc6c 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -265,6 +265,8 @@ type FakeWorkspace struct { // Operations belongs to MaterializationSets: the operations its handlers return, keyed by name. Operations map[string]ml.Operation + FeatureSpecs map[string]ml.FeatureSpec + SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value Acls map[string][]workspace.AclItem @@ -545,6 +547,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Features: map[string]ml.Feature{}, MaterializationSets: map[string]ml.MaterializationSet{}, Operations: map[string]ml.Operation{}, + FeatureSpecs: map[string]ml.FeatureSpec{}, Volumes: map[string]catalog.VolumeInfo{}, Dashboards: NewEventualMap[string, *fakeDashboard](eventualConsistency), PublishedDashboards: map[string]dashboards.PublishedDashboard{}, diff --git a/libs/testserver/feature_spec.go b/libs/testserver/feature_spec.go new file mode 100644 index 00000000000..61670dbd0d2 --- /dev/null +++ b/libs/testserver/feature_spec.go @@ -0,0 +1,63 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// FeatureSpecsCreate fakes POST /api/2.0/feature-engineering/feature-specs. The FeatureSpec body is +// sent directly, keyed by name, which is also the resource id. Like the real endpoint, it returns +// the feature spec it was sent, with the name lowercased. +func (s *FakeWorkspace) FeatureSpecsCreate(req Request) Response { + defer s.LockUnlock()() + + var featureSpec ml.FeatureSpec + if err := json.Unmarshal(req.Body, &featureSpec); err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + + // UC normalizes feature spec names to lowercase. + featureSpec.Name = strings.ToLower(featureSpec.Name) + + if len(strings.Split(featureSpec.Name, ".")) != 3 { + return Response{ + StatusCode: http.StatusBadRequest, + Body: fmt.Sprintf("name must be a three-part name, got %q", featureSpec.Name), + } + } + if _, exists := s.FeatureSpecs[featureSpec.Name]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Function '%s' already exists", featureSpec.Name)}, + } + } + + s.FeatureSpecs[featureSpec.Name] = featureSpec + return Response{ + Body: featureSpec, + } +} + +// FeatureSpecsDelete fakes DELETE /api/2.0/feature-engineering/feature-specs/{name}. A feature spec +// is a UC function, and UC drops a function's grants with it, so a recreated feature spec starts +// with none. +func (s *FakeWorkspace) FeatureSpecsDelete(name string) Response { + defer s.LockUnlock()() + + if _, ok := s.FeatureSpecs[name]; !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": fmt.Sprintf("Function '%s' does not exist.", name)}, + } + } + delete(s.FeatureSpecs, name) + delete(s.Grants, grantsKey("function", name)) + return Response{Body: map[string]any{}} +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e5f4c37a16b..8b58b6cde47 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -957,6 +957,17 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.MaterializationSetsDelete(req.Vars["materialization_set_id"]) }) + // Feature Engineering feature specs: + server.Handle("POST", "/api/2.0/feature-engineering/feature-specs", func(req Request) any { + return req.Workspace.FeatureSpecsCreate(req) + }) + server.Handle("GET", "/api/2.0/feature-engineering/feature-specs/{name}", func(req Request) any { + return MapGet(req.Workspace, req.Workspace.FeatureSpecs, req.Vars["name"]) + }) + server.Handle("DELETE", "/api/2.0/feature-engineering/feature-specs/{name}", func(req Request) any { + return req.Workspace.FeatureSpecsDelete(req.Vars["name"]) + }) + // Model Provider Services (AI Gateway): server.Handle("POST", "/api/2.1/unity-catalog/model-provider-services", func(req Request) any { diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 0c31a7a0071..2773cb792d8 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -19,6 +19,7 @@ var resourceURLPatterns = map[string]string{ "database_instances": "compute/database-instances/%s", "experiments": "ml/experiments/%s", "features": "explore/data/features/%s", + "feature_specs": "explore/data/functions/%s", "genie_spaces": "genie/rooms/%s", "jobs": "jobs/%s", "mcp_services": "explore/data/mcp-services/%s", @@ -58,6 +59,7 @@ var resourceAliases = map[string]string{ var dotSeparatedResources = map[string]bool{ "catalogs": true, "features": true, + "feature_specs": true, "mcp_services": true, "model_services": true, "model_provider_services": true, diff --git a/python/databricks/bundles/core/__init__.py b/python/databricks/bundles/core/__init__.py index 2c38b5f19b9..cee232d1704 100644 --- a/python/databricks/bundles/core/__init__.py +++ b/python/databricks/bundles/core/__init__.py @@ -25,6 +25,7 @@ "database_instance_mutator", "external_location_mutator", "feature_mutator", + "feature_spec_mutator", "genie_space_mutator", "instance_pool_mutator", "job_mutator", @@ -72,6 +73,7 @@ database_instance_mutator, external_location_mutator, feature_mutator, + feature_spec_mutator, genie_space_mutator, instance_pool_mutator, job_mutator, diff --git a/python/databricks/bundles/core/_generated/__init__.py b/python/databricks/bundles/core/_generated/__init__.py index 1d35d7b81ff..7533a36dda3 100644 --- a/python/databricks/bundles/core/_generated/__init__.py +++ b/python/databricks/bundles/core/_generated/__init__.py @@ -40,6 +40,10 @@ _ExternalLocationResources, external_location_mutator, ) +from databricks.bundles.core._generated.feature_specs import ( + _FeatureSpecResources, + feature_spec_mutator, +) from databricks.bundles.core._generated.features import ( _FeatureResources, feature_mutator, @@ -134,6 +138,7 @@ "database_instance_mutator", "external_location_mutator", "feature_mutator", + "feature_spec_mutator", "genie_space_mutator", "instance_pool_mutator", "job_mutator", @@ -171,6 +176,7 @@ class _GeneratedResources( _DatabaseInstanceResources, _MlflowExperimentResources, _ExternalLocationResources, + _FeatureSpecResources, _FeatureResources, _GenieSpaceResources, _InstancePoolResources, @@ -210,6 +216,7 @@ def _all_resource_types() -> "tuple[_ResourceType, ...]": database_instances, experiments, external_locations, + feature_specs, features, genie_spaces, instance_pools, @@ -246,6 +253,7 @@ def _all_resource_types() -> "tuple[_ResourceType, ...]": database_instances._resource_type(), experiments._resource_type(), external_locations._resource_type(), + feature_specs._resource_type(), features._resource_type(), genie_spaces._resource_type(), instance_pools._resource_type(), diff --git a/python/databricks/bundles/core/_generated/feature_specs.py b/python/databricks/bundles/core/_generated/feature_specs.py new file mode 100644 index 00000000000..7d82ac27ed3 --- /dev/null +++ b/python/databricks/bundles/core/_generated/feature_specs.py @@ -0,0 +1,118 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.feature_specs._models.feature_spec import ( + FeatureSpec, + FeatureSpecParam, + ) + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.feature_specs._models.feature_spec import FeatureSpec + + return _ResourceType( + resource_type=FeatureSpec, + singular_name="feature_spec", + plural_name="feature_specs", + ) + + +class _FeatureSpecResources: + """ + Generated feature_spec accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def feature_specs(self) -> dict[str, "FeatureSpec"]: + return self._resources["feature_specs"] + + def add_feature_spec( + self, + resource_name: str, + feature_spec: "FeatureSpecParam", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource feature_spec to the collection of resources. Resource name must be unique across all feature_specs. + + :param resource_name: unique identifier for the feature_spec + :param feature_spec: the feature_spec to add, can be FeatureSpec or dict + :param location: optional location of the feature_spec in the source code + """ + from databricks.bundles.feature_specs._models.feature_spec import FeatureSpec + + feature_spec = _transform(FeatureSpec, feature_spec) + path = ("resources", "feature_specs", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["feature_specs"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource 'feature_spec'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["feature_specs"][resource_name] = feature_spec + + +@overload +def feature_spec_mutator( + function: Callable[[Bundle, "FeatureSpec"], "FeatureSpec"], +) -> ResourceMutator["FeatureSpec"]: ... + + +@overload +def feature_spec_mutator( + function: Callable[["FeatureSpec"], "FeatureSpec"], +) -> ResourceMutator["FeatureSpec"]: ... + + +def feature_spec_mutator(function: Callable) -> ResourceMutator["FeatureSpec"]: + """ + Decorator for defining mutator for feature_specs. Function should return a new instance of the feature_spec + with the desired changes, instead of mutating the input feature_spec. + + Example: + + .. code-block:: python + + @feature_spec_mutator + def my_feature_spec_mutator(bundle: Bundle, feature_spec: FeatureSpec) -> FeatureSpec: + return replace(feature_spec, ...) + + :param function: Function that mutates feature_specs. + """ + from databricks.bundles.feature_specs._models.feature_spec import FeatureSpec + + return ResourceMutator(resource_type=FeatureSpec, function=function) diff --git a/python/databricks/bundles/feature_specs/__init__.py b/python/databricks/bundles/feature_specs/__init__.py new file mode 100644 index 00000000000..fa6da07eb61 --- /dev/null +++ b/python/databricks/bundles/feature_specs/__init__.py @@ -0,0 +1,105 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +__all__ = [ + "DeclarativeFeature", + "DeclarativeFeatureDict", + "DeclarativeFeatureParam", + "DefaultValue", + "DefaultValueDict", + "DefaultValueParam", + "DependsOn", + "DependsOnDict", + "DependsOnParam", + "FeatureFunction", + "FeatureFunctionDict", + "FeatureFunctionInputBinding", + "FeatureFunctionInputBindingDict", + "FeatureFunctionInputBindingParam", + "FeatureFunctionParam", + "FeatureLookup", + "FeatureLookupDict", + "FeatureLookupParam", + "FeatureSpec", + "FeatureSpecDict", + "FeatureSpecParam", + "Features", + "FeaturesDict", + "FeaturesParam", + "Lifecycle", + "LifecycleDict", + "LifecycleParam", + "PipRequirements", + "PipRequirementsDict", + "PipRequirementsParam", + "Privilege", + "PrivilegeAssignment", + "PrivilegeAssignmentDict", + "PrivilegeAssignmentParam", + "PrivilegeParam", + "RenameOutput", + "RenameOutputDict", + "RenameOutputParam", +] + + +from databricks.bundles.feature_specs._models.declarative_feature import ( + DeclarativeFeature, + DeclarativeFeatureDict, + DeclarativeFeatureParam, +) +from databricks.bundles.feature_specs._models.default_value import ( + DefaultValue, + DefaultValueDict, + DefaultValueParam, +) +from databricks.bundles.feature_specs._models.depends_on import ( + DependsOn, + DependsOnDict, + DependsOnParam, +) +from databricks.bundles.feature_specs._models.feature_function import ( + FeatureFunction, + FeatureFunctionDict, + FeatureFunctionParam, +) +from databricks.bundles.feature_specs._models.feature_function_input_binding import ( + FeatureFunctionInputBinding, + FeatureFunctionInputBindingDict, + FeatureFunctionInputBindingParam, +) +from databricks.bundles.feature_specs._models.feature_lookup import ( + FeatureLookup, + FeatureLookupDict, + FeatureLookupParam, +) +from databricks.bundles.feature_specs._models.feature_spec import ( + FeatureSpec, + FeatureSpecDict, + FeatureSpecParam, +) +from databricks.bundles.feature_specs._models.features import ( + Features, + FeaturesDict, + FeaturesParam, +) +from databricks.bundles.feature_specs._models.lifecycle import ( + Lifecycle, + LifecycleDict, + LifecycleParam, +) +from databricks.bundles.feature_specs._models.pip_requirements import ( + PipRequirements, + PipRequirementsDict, + PipRequirementsParam, +) +from databricks.bundles.feature_specs._models.privilege import Privilege, PrivilegeParam +from databricks.bundles.feature_specs._models.privilege_assignment import ( + PrivilegeAssignment, + PrivilegeAssignmentDict, + PrivilegeAssignmentParam, +) +from databricks.bundles.feature_specs._models.rename_output import ( + RenameOutput, + RenameOutputDict, + RenameOutputParam, +) diff --git a/python/databricks/bundles/feature_specs/_models/declarative_feature.py b/python/databricks/bundles/feature_specs/_models/declarative_feature.py new file mode 100644 index 00000000000..f5a13764d30 --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/declarative_feature.py @@ -0,0 +1,48 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class DeclarativeFeature: + """ + :meta private: [EXPERIMENTAL] + + A reference to a Feature. + """ + + feature: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Three-part Unity Catalog name (catalog.schema.feature) of the Feature. + """ + + @classmethod + def from_dict(cls, value: "DeclarativeFeatureDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "DeclarativeFeatureDict": + return _transform_to_json_value(self) # type:ignore + + +class DeclarativeFeatureDict(TypedDict, total=False): + """""" + + feature: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Three-part Unity Catalog name (catalog.schema.feature) of the Feature. + """ + + +DeclarativeFeatureParam = DeclarativeFeatureDict | DeclarativeFeature diff --git a/python/databricks/bundles/feature_specs/_models/default_value.py b/python/databricks/bundles/feature_specs/_models/default_value.py new file mode 100644 index 00000000000..88e1491fd46 --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/default_value.py @@ -0,0 +1,62 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr, VariableOrOptional + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class DefaultValue: + """ + :meta private: [EXPERIMENTAL] + + A default value for a looked-up feature. + """ + + feature: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Name of the feature: its output name if it is renamed, otherwise its name in the feature table. + """ + + default_value: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Default value for the feature. + """ + + @classmethod + def from_dict(cls, value: "DefaultValueDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "DefaultValueDict": + return _transform_to_json_value(self) # type:ignore + + +class DefaultValueDict(TypedDict, total=False): + """""" + + feature: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Name of the feature: its output name if it is renamed, otherwise its name in the feature table. + """ + + default_value: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Default value for the feature. + """ + + +DefaultValueParam = DefaultValueDict | DefaultValue diff --git a/python/databricks/bundles/feature_specs/_models/depends_on.py b/python/databricks/bundles/feature_specs/_models/depends_on.py new file mode 100644 index 00000000000..92fe3d5ea1e --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/depends_on.py @@ -0,0 +1,40 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class DependsOn: + """""" + + resource: VariableOr[str] + """ + Key of the resource to depend on, as `foo` or `jobs.foo`. + """ + + @classmethod + def from_dict(cls, value: "DependsOnDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "DependsOnDict": + return _transform_to_json_value(self) # type:ignore + + +class DependsOnDict(TypedDict, total=False): + """""" + + resource: VariableOr[str] + """ + Key of the resource to depend on, as `foo` or `jobs.foo`. + """ + + +DependsOnParam = DependsOnDict | DependsOn diff --git a/python/databricks/bundles/feature_specs/_models/feature_function.py b/python/databricks/bundles/feature_specs/_models/feature_function.py new file mode 100644 index 00000000000..effde4418ea --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/feature_function.py @@ -0,0 +1,86 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import ( + VariableOr, + VariableOrList, + VariableOrOptional, +) +from databricks.bundles.feature_specs._models.feature_function_input_binding import ( + FeatureFunctionInputBinding, + FeatureFunctionInputBindingParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class FeatureFunction: + """ + :meta private: [EXPERIMENTAL] + + A Python UDF that computes a feature. + """ + + udf: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Three-part name of the Python UDF. + """ + + input_bindings: VariableOrList[FeatureFunctionInputBinding] = field( + default_factory=list + ) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Bindings of the UDF's parameters to features. + """ + + output: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Output name of the feature that the UDF computes. + """ + + @classmethod + def from_dict(cls, value: "FeatureFunctionDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "FeatureFunctionDict": + return _transform_to_json_value(self) # type:ignore + + +class FeatureFunctionDict(TypedDict, total=False): + """""" + + udf: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Three-part name of the Python UDF. + """ + + input_bindings: VariableOrList[FeatureFunctionInputBindingParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Bindings of the UDF's parameters to features. + """ + + output: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Output name of the feature that the UDF computes. + """ + + +FeatureFunctionParam = FeatureFunctionDict | FeatureFunction diff --git a/python/databricks/bundles/feature_specs/_models/feature_function_input_binding.py b/python/databricks/bundles/feature_specs/_models/feature_function_input_binding.py new file mode 100644 index 00000000000..6a28c15c6cc --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/feature_function_input_binding.py @@ -0,0 +1,64 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class FeatureFunctionInputBinding: + """ + :meta private: [EXPERIMENTAL] + + Binds one parameter of the UDF to a feature. + """ + + bound_feature: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Name of the feature bound to the parameter. + """ + + parameter: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Name of the UDF parameter. + """ + + @classmethod + def from_dict(cls, value: "FeatureFunctionInputBindingDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "FeatureFunctionInputBindingDict": + return _transform_to_json_value(self) # type:ignore + + +class FeatureFunctionInputBindingDict(TypedDict, total=False): + """""" + + bound_feature: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Name of the feature bound to the parameter. + """ + + parameter: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Name of the UDF parameter. + """ + + +FeatureFunctionInputBindingParam = ( + FeatureFunctionInputBindingDict | FeatureFunctionInputBinding +) diff --git a/python/databricks/bundles/feature_specs/_models/feature_lookup.py b/python/databricks/bundles/feature_specs/_models/feature_lookup.py new file mode 100644 index 00000000000..49a7387e7e0 --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/feature_lookup.py @@ -0,0 +1,144 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import ( + VariableOr, + VariableOrList, + VariableOrOptional, +) +from databricks.bundles.feature_specs._models.default_value import ( + DefaultValue, + DefaultValueParam, +) +from databricks.bundles.feature_specs._models.rename_output import ( + RenameOutput, + RenameOutputParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class FeatureLookup: + """ + :meta private: [EXPERIMENTAL] + + Features to look up from a feature table. + """ + + table: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Three-part name of the feature table. + """ + + default_values: VariableOrList[DefaultValue] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Default values for features. + """ + + feature_names: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Names of the features to look up, or empty to look up every feature in the table. + """ + + lookback_window: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Lookback window to use when performing point-in-time lookup. + """ + + lookup_keys: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Keys to use when joining with the feature table. + """ + + rename_outputs: VariableOrList[RenameOutput] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Output names for features, in place of their names in the feature table. + """ + + timestamp_lookup_keys: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Keys to use when performing point-in-time lookup. + """ + + @classmethod + def from_dict(cls, value: "FeatureLookupDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "FeatureLookupDict": + return _transform_to_json_value(self) # type:ignore + + +class FeatureLookupDict(TypedDict, total=False): + """""" + + table: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Three-part name of the feature table. + """ + + default_values: VariableOrList[DefaultValueParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Default values for features. + """ + + feature_names: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Names of the features to look up, or empty to look up every feature in the table. + """ + + lookback_window: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Lookback window to use when performing point-in-time lookup. + """ + + lookup_keys: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Keys to use when joining with the feature table. + """ + + rename_outputs: VariableOrList[RenameOutputParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Output names for features, in place of their names in the feature table. + """ + + timestamp_lookup_keys: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Keys to use when performing point-in-time lookup. + """ + + +FeatureLookupParam = FeatureLookupDict | FeatureLookup diff --git a/python/databricks/bundles/feature_specs/_models/feature_spec.py b/python/databricks/bundles/feature_specs/_models/feature_spec.py new file mode 100644 index 00000000000..7aa9f9065c6 --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/feature_spec.py @@ -0,0 +1,129 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._resource import Resource +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import ( + VariableOr, + VariableOrList, + VariableOrOptional, +) +from databricks.bundles.feature_specs._models.features import Features, FeaturesParam +from databricks.bundles.feature_specs._models.lifecycle import Lifecycle, LifecycleParam +from databricks.bundles.feature_specs._models.pip_requirements import ( + PipRequirements, + PipRequirementsParam, +) +from databricks.bundles.feature_specs._models.privilege_assignment import ( + PrivilegeAssignment, + PrivilegeAssignmentParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class FeatureSpec(Resource): + """ + A named set of features to serve, stored as a Unity Catalog function. + """ + + name: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Three-part name (catalog.schema.feature_spec) of the feature spec. + """ + + exclude_columns: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Output names of columns to leave out of the feature spec. + """ + + extra_pip_requirements: VariableOrOptional[PipRequirements] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Extra pip requirements to install in the serving container, in addition to the defaults and + those resolved from the features. + """ + + features: VariableOrList[Features] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The Features (either a declarative one, or a lookup from a feature table, or a feature + function) to serve. A declarative feature (`DeclarativeFeature`) cannot be combined with + lookups (`FeatureLookup`) and functions (`FeatureFunction`) in one feature spec. + """ + + grants: VariableOrList[PrivilegeAssignment] = field(default_factory=list) + """ + The Unity Catalog privileges to grant to principals on this securable. + """ + + lifecycle: VariableOrOptional[Lifecycle] = None + """ + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + """ + + @classmethod + def from_dict(cls, value: "FeatureSpecDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "FeatureSpecDict": + return _transform_to_json_value(self) # type:ignore + + +class FeatureSpecDict(TypedDict, total=False): + """""" + + name: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Three-part name (catalog.schema.feature_spec) of the feature spec. + """ + + exclude_columns: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Output names of columns to leave out of the feature spec. + """ + + extra_pip_requirements: VariableOrOptional[PipRequirementsParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Extra pip requirements to install in the serving container, in addition to the defaults and + those resolved from the features. + """ + + features: VariableOrList[FeaturesParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The Features (either a declarative one, or a lookup from a feature table, or a feature + function) to serve. A declarative feature (`DeclarativeFeature`) cannot be combined with + lookups (`FeatureLookup`) and functions (`FeatureFunction`) in one feature spec. + """ + + grants: VariableOrList[PrivilegeAssignmentParam] + """ + The Unity Catalog privileges to grant to principals on this securable. + """ + + lifecycle: VariableOrOptional[LifecycleParam] + """ + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + """ + + +FeatureSpecParam = FeatureSpecDict | FeatureSpec diff --git a/python/databricks/bundles/feature_specs/_models/features.py b/python/databricks/bundles/feature_specs/_models/features.py new file mode 100644 index 00000000000..9aac3c11978 --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/features.py @@ -0,0 +1,88 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.feature_specs._models.declarative_feature import ( + DeclarativeFeature, + DeclarativeFeatureParam, +) +from databricks.bundles.feature_specs._models.feature_function import ( + FeatureFunction, + FeatureFunctionParam, +) +from databricks.bundles.feature_specs._models.feature_lookup import ( + FeatureLookup, + FeatureLookupParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class Features: + """ + :meta private: [EXPERIMENTAL] + + One entry of a feature spec: a Feature, a feature table lookup, or a feature function. + """ + + declarative_feature: VariableOrOptional[DeclarativeFeature] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] A Feature, referenced by its three-part Unity Catalog name. + """ + + feature_function: VariableOrOptional[FeatureFunction] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] A feature function. + """ + + feature_lookup: VariableOrOptional[FeatureLookup] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] A feature table lookup. + """ + + @classmethod + def from_dict(cls, value: "FeaturesDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "FeaturesDict": + return _transform_to_json_value(self) # type:ignore + + +class FeaturesDict(TypedDict, total=False): + """""" + + declarative_feature: VariableOrOptional[DeclarativeFeatureParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] A Feature, referenced by its three-part Unity Catalog name. + """ + + feature_function: VariableOrOptional[FeatureFunctionParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] A feature function. + """ + + feature_lookup: VariableOrOptional[FeatureLookupParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] A feature table lookup. + """ + + +FeaturesParam = FeaturesDict | Features diff --git a/python/databricks/bundles/feature_specs/_models/lifecycle.py b/python/databricks/bundles/feature_specs/_models/lifecycle.py new file mode 100644 index 00000000000..87952fededb --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/lifecycle.py @@ -0,0 +1,54 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.feature_specs._models.depends_on import ( + DependsOn, + DependsOnParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class Lifecycle: + """""" + + depends_on: VariableOrList[DependsOn] = field(default_factory=list) + """ + Resources that must be deployed before this resource and deleted after it. Unlike variable references, this only affects ordering. + """ + + prevent_destroy: VariableOrOptional[bool] = None + """ + Lifecycle setting to prevent the resource from being destroyed. + """ + + @classmethod + def from_dict(cls, value: "LifecycleDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "LifecycleDict": + return _transform_to_json_value(self) # type:ignore + + +class LifecycleDict(TypedDict, total=False): + """""" + + depends_on: VariableOrList[DependsOnParam] + """ + Resources that must be deployed before this resource and deleted after it. Unlike variable references, this only affects ordering. + """ + + prevent_destroy: VariableOrOptional[bool] + """ + Lifecycle setting to prevent the resource from being destroyed. + """ + + +LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/feature_specs/_models/pip_requirements.py b/python/databricks/bundles/feature_specs/_models/pip_requirements.py new file mode 100644 index 00000000000..2c5a597a2ab --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/pip_requirements.py @@ -0,0 +1,48 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipRequirements: + """ + :meta private: [EXPERIMENTAL] + + Pip requirements to install in a serving container. + """ + + requirements: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Pip requirement strings (PEP 508) or UC Volume whl paths (/Volumes/...). + """ + + @classmethod + def from_dict(cls, value: "PipRequirementsDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipRequirementsDict": + return _transform_to_json_value(self) # type:ignore + + +class PipRequirementsDict(TypedDict, total=False): + """""" + + requirements: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Pip requirement strings (PEP 508) or UC Volume whl paths (/Volumes/...). + """ + + +PipRequirementsParam = PipRequirementsDict | PipRequirements diff --git a/python/databricks/bundles/feature_specs/_models/privilege.py b/python/databricks/bundles/feature_specs/_models/privilege.py new file mode 100644 index 00000000000..382f3917d5a --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/privilege.py @@ -0,0 +1,134 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class Privilege(Enum): + SELECT = "SELECT" + READ_PRIVATE_FILES = "READ_PRIVATE_FILES" + WRITE_PRIVATE_FILES = "WRITE_PRIVATE_FILES" + CREATE = "CREATE" + USAGE = "USAGE" + USE_CATALOG = "USE_CATALOG" + USE_SCHEMA = "USE_SCHEMA" + CREATE_SCHEMA = "CREATE_SCHEMA" + CREATE_VIEW = "CREATE_VIEW" + CREATE_EXTERNAL_TABLE = "CREATE_EXTERNAL_TABLE" + CREATE_MATERIALIZED_VIEW = "CREATE_MATERIALIZED_VIEW" + CREATE_FUNCTION = "CREATE_FUNCTION" + CREATE_MODEL = "CREATE_MODEL" + CREATE_CATALOG = "CREATE_CATALOG" + CREATE_MANAGED_STORAGE = "CREATE_MANAGED_STORAGE" + CREATE_EXTERNAL_LOCATION = "CREATE_EXTERNAL_LOCATION" + CREATE_STORAGE_CREDENTIAL = "CREATE_STORAGE_CREDENTIAL" + CREATE_SERVICE_CREDENTIAL = "CREATE_SERVICE_CREDENTIAL" + ACCESS = "ACCESS" + CREATE_SHARE = "CREATE_SHARE" + CREATE_RECIPIENT = "CREATE_RECIPIENT" + CREATE_PROVIDER = "CREATE_PROVIDER" + USE_SHARE = "USE_SHARE" + USE_RECIPIENT = "USE_RECIPIENT" + USE_PROVIDER = "USE_PROVIDER" + USE_MARKETPLACE_ASSETS = "USE_MARKETPLACE_ASSETS" + SET_SHARE_PERMISSION = "SET_SHARE_PERMISSION" + MODIFY = "MODIFY" + REFRESH = "REFRESH" + EXECUTE = "EXECUTE" + READ_FILES = "READ_FILES" + WRITE_FILES = "WRITE_FILES" + CREATE_TABLE = "CREATE_TABLE" + ALL_PRIVILEGES = "ALL_PRIVILEGES" + CREATE_CONNECTION = "CREATE_CONNECTION" + USE_CONNECTION = "USE_CONNECTION" + APPLY_TAG = "APPLY_TAG" + CREATE_FOREIGN_CATALOG = "CREATE_FOREIGN_CATALOG" + CREATE_FOREIGN_SECURABLE = "CREATE_FOREIGN_SECURABLE" + MANAGE_ALLOWLIST = "MANAGE_ALLOWLIST" + CREATE_VOLUME = "CREATE_VOLUME" + CREATE_EXTERNAL_VOLUME = "CREATE_EXTERNAL_VOLUME" + READ_VOLUME = "READ_VOLUME" + WRITE_VOLUME = "WRITE_VOLUME" + MANAGE = "MANAGE" + BROWSE = "BROWSE" + CREATE_CLEAN_ROOM = "CREATE_CLEAN_ROOM" + MODIFY_CLEAN_ROOM = "MODIFY_CLEAN_ROOM" + EXECUTE_CLEAN_ROOM_TASK = "EXECUTE_CLEAN_ROOM_TASK" + EXTERNAL_USE_SCHEMA = "EXTERNAL_USE_SCHEMA" + READ_METADATA = "READ_METADATA" + CREATE_FEATURE = "CREATE_FEATURE" + READ_FEATURE = "READ_FEATURE" + EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + + +PrivilegeParam = ( + Literal[ + "SELECT", + "READ_PRIVATE_FILES", + "WRITE_PRIVATE_FILES", + "CREATE", + "USAGE", + "USE_CATALOG", + "USE_SCHEMA", + "CREATE_SCHEMA", + "CREATE_VIEW", + "CREATE_EXTERNAL_TABLE", + "CREATE_MATERIALIZED_VIEW", + "CREATE_FUNCTION", + "CREATE_MODEL", + "CREATE_CATALOG", + "CREATE_MANAGED_STORAGE", + "CREATE_EXTERNAL_LOCATION", + "CREATE_STORAGE_CREDENTIAL", + "CREATE_SERVICE_CREDENTIAL", + "ACCESS", + "CREATE_SHARE", + "CREATE_RECIPIENT", + "CREATE_PROVIDER", + "USE_SHARE", + "USE_RECIPIENT", + "USE_PROVIDER", + "USE_MARKETPLACE_ASSETS", + "SET_SHARE_PERMISSION", + "MODIFY", + "REFRESH", + "EXECUTE", + "READ_FILES", + "WRITE_FILES", + "CREATE_TABLE", + "ALL_PRIVILEGES", + "CREATE_CONNECTION", + "USE_CONNECTION", + "APPLY_TAG", + "CREATE_FOREIGN_CATALOG", + "CREATE_FOREIGN_SECURABLE", + "MANAGE_ALLOWLIST", + "CREATE_VOLUME", + "CREATE_EXTERNAL_VOLUME", + "READ_VOLUME", + "WRITE_VOLUME", + "MANAGE", + "BROWSE", + "CREATE_CLEAN_ROOM", + "MODIFY_CLEAN_ROOM", + "EXECUTE_CLEAN_ROOM_TASK", + "EXTERNAL_USE_SCHEMA", + "READ_METADATA", + "CREATE_FEATURE", + "READ_FEATURE", + "EXTERNAL_USE_LOCATION", + ] + | Privilege + | str +) diff --git a/python/databricks/bundles/feature_specs/_models/privilege_assignment.py b/python/databricks/bundles/feature_specs/_models/privilege_assignment.py new file mode 100644 index 00000000000..be373a00556 --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/privilege_assignment.py @@ -0,0 +1,53 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.feature_specs._models.privilege import Privilege, PrivilegeParam + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PrivilegeAssignment: + """""" + + principal: VariableOrOptional[str] = None + """ + The principal (user email address or group name). + For deleted principals, `principal` is empty while `principal_id` is populated. + """ + + privileges: VariableOrList[Privilege] = field(default_factory=list) + """ + The privileges assigned to the principal. + """ + + @classmethod + def from_dict(cls, value: "PrivilegeAssignmentDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PrivilegeAssignmentDict": + return _transform_to_json_value(self) # type:ignore + + +class PrivilegeAssignmentDict(TypedDict, total=False): + """""" + + principal: VariableOrOptional[str] + """ + The principal (user email address or group name). + For deleted principals, `principal` is empty while `principal_id` is populated. + """ + + privileges: VariableOrList[PrivilegeParam] + """ + The privileges assigned to the principal. + """ + + +PrivilegeAssignmentParam = PrivilegeAssignmentDict | PrivilegeAssignment diff --git a/python/databricks/bundles/feature_specs/_models/rename_output.py b/python/databricks/bundles/feature_specs/_models/rename_output.py new file mode 100644 index 00000000000..631972cb052 --- /dev/null +++ b/python/databricks/bundles/feature_specs/_models/rename_output.py @@ -0,0 +1,62 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class RenameOutput: + """ + :meta private: [EXPERIMENTAL] + + An output name for a looked-up feature. + """ + + feature: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Name of the feature in the feature table. + """ + + output: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Output name for the feature. + """ + + @classmethod + def from_dict(cls, value: "RenameOutputDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "RenameOutputDict": + return _transform_to_json_value(self) # type:ignore + + +class RenameOutputDict(TypedDict, total=False): + """""" + + feature: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Name of the feature in the feature table. + """ + + output: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Output name for the feature. + """ + + +RenameOutputParam = RenameOutputDict | RenameOutput diff --git a/python/databricks_tests/core/_generated/__init__.py b/python/databricks_tests/core/_generated/__init__.py index 63fb4cf0c6b..3382c34d83b 100644 --- a/python/databricks_tests/core/_generated/__init__.py +++ b/python/databricks_tests/core/_generated/__init__.py @@ -12,6 +12,7 @@ database_instances, experiments, external_locations, + feature_specs, features, genie_spaces, instance_pools, @@ -50,6 +51,7 @@ database_instances._test_case(), experiments._test_case(), external_locations._test_case(), + feature_specs._test_case(), features._test_case(), genie_spaces._test_case(), instance_pools._test_case(), diff --git a/python/databricks_tests/core/_generated/feature_specs.py b/python/databricks_tests/core/_generated/feature_specs.py new file mode 100644 index 00000000000..b8114f298a5 --- /dev/null +++ b/python/databricks_tests/core/_generated/feature_specs.py @@ -0,0 +1,24 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from databricks.bundles.core import Resources, feature_spec_mutator +from databricks.bundles.core._generated.feature_specs import _resource_type +from databricks.bundles.feature_specs._models.feature_spec import FeatureSpec +from databricks.bundles.feature_specs._models.lifecycle import Lifecycle +from databricks.bundles.feature_specs._models.privilege_assignment import ( + PrivilegeAssignment, +) +from databricks_tests.core._resource_test_case import ResourceTestCase + + +def _test_case(): + return ( + ResourceTestCase( + add_resource=Resources.add_feature_spec, + dict_example={"grants": [{}], "lifecycle": {}, "name": "name"}, + dataclass_example=FeatureSpec( + grants=[PrivilegeAssignment()], lifecycle=Lifecycle(), name="name" + ), + mutator=feature_spec_mutator, + ), + _resource_type(), + ) diff --git a/python/databricks_tests/core/public_api.txt b/python/databricks_tests/core/public_api.txt index 9bfea8bf0b1..294db70084a 100644 --- a/python/databricks_tests/core/public_api.txt +++ b/python/databricks_tests/core/public_api.txt @@ -24,6 +24,7 @@ __all__ = [ database_instance_mutator, external_location_mutator, feature_mutator, + feature_spec_mutator, genie_space_mutator, instance_pool_mutator, job_mutator, @@ -106,6 +107,7 @@ class Resources: def add_diagnostics(self, other: Diagnostics) -> None def add_external_location(self, resource_name: str, external_location: ExternalLocationParam, *, location: Union[Location, None] = None) -> None def add_feature(self, resource_name: str, feature: FeatureParam, *, location: Union[Location, None] = None) -> None + def add_feature_spec(self, resource_name: str, feature_spec: FeatureSpecParam, *, location: Union[Location, None] = None) -> None def add_genie_space(self, resource_name: str, genie_space: GenieSpaceParam, *, location: Union[Location, None] = None) -> None def add_instance_pool(self, resource_name: str, instance_pool: InstancePoolParam, *, location: Union[Location, None] = None) -> None def add_job(self, resource_name: str, job: JobParam, *, location: Union[Location, None] = None) -> None @@ -143,6 +145,7 @@ class Resources: @property diagnostics -> Diagnostics @property experiments -> dict[str, MlflowExperiment] @property external_locations -> dict[str, ExternalLocation] + @property feature_specs -> dict[str, FeatureSpec] @property features -> dict[str, Feature] @property genie_spaces -> dict[str, GenieSpace] @property instance_pools -> dict[str, InstancePool] @@ -227,6 +230,10 @@ def external_location_mutator(function: Callable) -> ResourceMutator[ExternalLoc @overload def feature_mutator(function: Callable[[Feature], Feature]) -> ResourceMutator[Feature] def feature_mutator(function: Callable) -> ResourceMutator[Feature] +@overload def feature_spec_mutator(function: Callable[[Bundle, FeatureSpec], FeatureSpec]) -> ResourceMutator[FeatureSpec] +@overload def feature_spec_mutator(function: Callable[[FeatureSpec], FeatureSpec]) -> ResourceMutator[FeatureSpec] +def feature_spec_mutator(function: Callable) -> ResourceMutator[FeatureSpec] + @overload def genie_space_mutator(function: Callable[[Bundle, GenieSpace], GenieSpace]) -> ResourceMutator[GenieSpace] @overload def genie_space_mutator(function: Callable[[GenieSpace], GenieSpace]) -> ResourceMutator[GenieSpace] def genie_space_mutator(function: Callable) -> ResourceMutator[GenieSpace] @@ -337,6 +344,7 @@ singular_name=database_catalog plural_name=database_catalogs resource_type=Datab singular_name=database_instance plural_name=database_instances resource_type=DatabaseInstance singular_name=external_location plural_name=external_locations resource_type=ExternalLocation singular_name=feature plural_name=features resource_type=Feature +singular_name=feature_spec plural_name=feature_specs resource_type=FeatureSpec singular_name=genie_space plural_name=genie_spaces resource_type=GenieSpace singular_name=instance_pool plural_name=instance_pools resource_type=InstancePool singular_name=job plural_name=jobs resource_type=Job diff --git a/python/docs/databricks.bundles.feature_specs.rst b/python/docs/databricks.bundles.feature_specs.rst new file mode 100644 index 00000000000..602170bf55a --- /dev/null +++ b/python/docs/databricks.bundles.feature_specs.rst @@ -0,0 +1,11 @@ +Feature Specs +=============================== + +.. currentmodule:: databricks.bundles.feature_specs + +**Package:** ``databricks.bundles.feature_specs`` + +Classes +--------------- + +.. automodule:: databricks.bundles.feature_specs diff --git a/python/docs/index.rst b/python/docs/index.rst index 365990b43cc..fd443ec9e5d 100644 --- a/python/docs/index.rst +++ b/python/docs/index.rst @@ -21,6 +21,7 @@ See `Bundle configuration in Python