diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml new file mode 100644 index 00000000000..09b1f8647e0 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml @@ -0,0 +1,25 @@ +bundle: + name: pipeline-testing-dynamic-version + +artifacts: + other_test_code: + type: whl + files: + - source: dist/other_test_code-0.0.1-py3-none-any.whl + dynamic_version: true + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - ./dist/other_test_code-0.0.1-py3-none-any.whl + - ./dist/other_test_code-0.0.1-py3-none-any.whl[test] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt b/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt new file mode 100644 index 00000000000..9af556df861 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt @@ -0,0 +1,27 @@ + +=== Test dependencies use the patched wheel, with extras kept + +>>> [CLI] bundle deploy +Uploading .databricks/bundle/default/patched_wheels/other_test_code_other_test_code/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/files... +Created pipelines.orders +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +[ + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl[test]" +] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script new file mode 100644 index 00000000000..e71c775893f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script @@ -0,0 +1,9 @@ +trap "rm -fr dist" EXIT +mkdir dist +cp $TESTDIR/../../../artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl dist + +title "Test dependencies use the patched wheel, with extras kept" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '.body.testing.environment.dependencies' +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml new file mode 100644 index 00000000000..25d8c36639f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml @@ -0,0 +1,4 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml new file mode 100644 index 00000000000..381095e1cd3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: pipeline-testing-same-name-libraries + +resources: + pipelines: + orders: + name: orders + libraries: + - file: + path: ./transformations/orders.py + environment: + dependencies: + - ./pipeline/test_helpers-0.1-py3-none-any.whl + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + # A different wheel with the same file name would be uploaded to the same path. + - ./tests/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt new file mode 100644 index 00000000000..68399999172 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt @@ -0,0 +1,10 @@ + +>>> musterr [CLI] bundle deploy +Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl + at resources.pipelines.orders.environment.dependencies[0] + resources.pipelines.orders.testing.environment.dependencies[0] + in databricks.yml:13:13 + databricks.yml:21:15 + +Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl + diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script new file mode 100644 index 00000000000..32f10d10630 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script @@ -0,0 +1 @@ +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml new file mode 100644 index 00000000000..d77028ab40a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -0,0 +1,6 @@ +# Runs only against the local test server. +Cloud = false + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index edb0eb1c49a..a1c2243b694 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -1,17 +1,19 @@ +=== Relative test paths resolve to the deployed workspace paths + >>> [CLI] bundle validate -o json { "environment": { "dependencies": [ - "-e .[test]", - "-r ./tests/requirements.txt", + "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", "./dist/*.whl" ] }, "sources": [ { "glob": { - "include": "./tests/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" } } ] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index b7f915f687c..e32724e5611 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -1 +1,2 @@ +title "Relative test paths resolve to the deployed workspace paths" trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..0824dbbd696 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -44,48 +44,42 @@ func pipelineRewritePatterns() []pipelineRewritePattern { mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + skipRewrite: noSkipRewrite, + }, } return allPatterns } func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { - pipelineEnvironmentsPatterns := []pipelineRewritePattern{ - { - pattern: dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - mode: TranslateModeLocalRelativeWithPrefix, - skipRewrite: func(s string) bool { - return !libraries.IsLibraryLocal(s) - }, - }, + // Test dependencies take the same forms as the pipeline's environment dependencies. + dependencies := []dyn.Pattern{ + base.Append(dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), } - pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ - { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - TranslateModeEnvironmentPipFlag, - func(s string) bool { + var patterns []pipelineRewritePattern + for _, pattern := range dependencies { + patterns = append(patterns, pipelineRewritePattern{ + pattern: pattern, + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, + }) + } + for _, pattern := range dependencies { + patterns = append(patterns, pipelineRewritePattern{ + pattern: pattern, + mode: TranslateModeEnvironmentPipFlag, + skipRewrite: func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) return !ok }, - }, + }) } - - return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) + return patterns } func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..6b527cf7942 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -39,6 +39,13 @@ func TestVisitPipelinePaths(t *testing.T) { "src/foo.whl", }, }, + // The acceptance test bundle/resources/pipelines/testing covers the testing paths + // that are rewritten. A package name is not a path. + Testing: &pipelines.PipelineTesting{ + Environment: &pipelines.PipelineTestEnvironment{ + Dependencies: []string{"faker"}, + }, + }, }, }, }, diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..91f5ab261b1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var pipelineTestingEnvDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("testing"), + dyn.Key("environment"), + dyn.Key("dependencies"), +) + var clusterLibrariesPattern = dyn.NewPattern( dyn.Key("resources"), dyn.Key("clusters"), @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: pipelineTestingEnvDepsPattern, + fn: expandEnvironmentDeps, + }, { pattern: clusterLibrariesPattern, fn: expandLibraries, diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index ff6be05c3b2..020fd5bbdaa 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -144,6 +144,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..e4e1c62c614 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{ clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), } type libData struct { diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 3c5ff184a97..0169f1adb3a 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -95,6 +95,24 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } + // Update resources.pipelines.*.testing.environment.dependencies[*] + for pipelineName, pipelineRef := range b.Config.Resources.Pipelines { + if pipelineRef == nil || pipelineRef.Testing == nil || pipelineRef.Testing.Environment == nil { + continue + } + deps := pipelineRef.Testing.Environment.Dependencies + for depInd, dep := range deps { + base, extras := patchwheel.SplitWheelExtras(dep) + repl := replacements[base] + if repl != "" { + log.Debugf(ctx, "Updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s to %s", pipelineName, depInd, dep, repl+extras) + deps[depInd] = repl + extras + } else { + log.Debugf(ctx, "Not updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s. Available replacements: %v", pipelineName, depInd, dep, slices.Sorted(maps.Keys(replacements))) + } + } + } + return nil }