From 565ee9be712da02dd3d560c26ccff6e4ade30946 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Wed, 11 Nov 2026 21:57:50 +0000 Subject: [PATCH 1/5] Translate the testing paths of pipelines and upload local test wheels Relative paths in testing.sources and testing.environment.dependencies resolve to their deployed workspace paths, like libraries and environment dependencies. Local wheels in test dependencies are uploaded like other libraries, and the same-name check covers them. The testing property is only accepted with DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true, so bundles without it are not changed. Co-authored-by: Isaac --- .../databricks.yml | 21 +++++++++++++++++++ .../testing-same-name-libraries/out.test.toml | 2 ++ .../testing-same-name-libraries/output.txt | 10 +++++++++ .../test_helpers-0.1-py3-none-any.whl | 0 .../testing-same-name-libraries/script | 1 + .../testing-same-name-libraries/test.toml | 7 +++++++ .../tests/test_helpers-0.1-py3-none-any.whl | 0 .../tests/test_orders.py | 2 ++ .../transformations/orders.py | 1 + .../dist/test_helpers-0.1-py3-none-any.whl | 0 .../resources/pipelines/testing/output.txt | 8 ++++--- .../bundle/resources/pipelines/testing/script | 1 + .../mutator/paths/pipeline_paths_visitor.go | 19 +++++++++++++++++ .../paths/pipeline_paths_visitor_test.go | 7 +++++++ bundle/libraries/expand_glob_references.go | 13 ++++++++++++ bundle/libraries/remote_path.go | 1 + bundle/libraries/same_name_libraries.go | 1 + 17 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/script create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml new file mode 100644 index 00000000000..381095e1cd3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: pipeline-testing-same-name-libraries + +resources: + pipelines: + orders: + name: orders + libraries: + - file: + path: ./transformations/orders.py + environment: + dependencies: + - ./pipeline/test_helpers-0.1-py3-none-any.whl + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + # A different wheel with the same file name would be uploaded to the same path. + - ./tests/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt new file mode 100644 index 00000000000..68399999172 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt @@ -0,0 +1,10 @@ + +>>> musterr [CLI] bundle deploy +Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl + at resources.pipelines.orders.environment.dependencies[0] + resources.pipelines.orders.testing.environment.dependencies[0] + in databricks.yml:13:13 + databricks.yml:21:15 + +Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl + diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script new file mode 100644 index 00000000000..32f10d10630 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script @@ -0,0 +1 @@ +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml new file mode 100644 index 00000000000..d018f5851ee --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -0,0 +1,7 @@ +# Runs only against the local test server. +Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index edb0eb1c49a..a1c2243b694 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -1,17 +1,19 @@ +=== Relative test paths resolve to the deployed workspace paths + >>> [CLI] bundle validate -o json { "environment": { "dependencies": [ - "-e .[test]", - "-r ./tests/requirements.txt", + "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", "./dist/*.whl" ] }, "sources": [ { "glob": { - "include": "./tests/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" } } ] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index b7f915f687c..e32724e5611 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -1 +1,2 @@ +title "Relative test paths resolve to the deployed workspace paths" trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..83be40afd7d 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -44,6 +44,11 @@ func pipelineRewritePatterns() []pipelineRewritePattern { mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + skipRewrite: noSkipRewrite, + }, } return allPatterns @@ -65,6 +70,12 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !libraries.IsLibraryLocal(s) }, }, + // Test dependencies take the same forms as the pipeline's environment dependencies. + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, + }, } pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ @@ -83,6 +94,14 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !ok }, }, + { + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + TranslateModeEnvironmentPipFlag, + func(s string) bool { + _, _, ok := libraries.IsLocalPathInPipFlag(s) + return !ok + }, + }, } return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..6b527cf7942 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -39,6 +39,13 @@ func TestVisitPipelinePaths(t *testing.T) { "src/foo.whl", }, }, + // The acceptance test bundle/resources/pipelines/testing covers the testing paths + // that are rewritten. A package name is not a path. + Testing: &pipelines.PipelineTesting{ + Environment: &pipelines.PipelineTestEnvironment{ + Dependencies: []string{"faker"}, + }, + }, }, }, }, diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..91f5ab261b1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var pipelineTestingEnvDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("testing"), + dyn.Key("environment"), + dyn.Key("dependencies"), +) + var clusterLibrariesPattern = dyn.NewPattern( dyn.Key("resources"), dyn.Key("clusters"), @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: pipelineTestingEnvDepsPattern, + fn: expandEnvironmentDeps, + }, { pattern: clusterLibrariesPattern, fn: expandLibraries, diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index ff6be05c3b2..020fd5bbdaa 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -144,6 +144,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..e4e1c62c614 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{ clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), } type libData struct { From d3a65c4bf7227a562aaf865754824a40cbcacd30 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 16 Nov 2026 23:55:08 +0000 Subject: [PATCH 2/5] Define the dependency path patterns of pipelines once for both locations environment.dependencies and testing.environment.dependencies share the translate modes and skip functions, so the patterns are built in one loop over the two locations. The order of the patterns does not change. Co-authored-by: Isaac --- .../mutator/paths/pipeline_paths_visitor.go | 61 ++++++------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index 83be40afd7d..0824dbbd696 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -55,56 +55,31 @@ func pipelineRewritePatterns() []pipelineRewritePattern { } func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { - pipelineEnvironmentsPatterns := []pipelineRewritePattern{ - { - pattern: dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - mode: TranslateModeLocalRelativeWithPrefix, - skipRewrite: func(s string) bool { - return !libraries.IsLibraryLocal(s) - }, - }, - // Test dependencies take the same forms as the pipeline's environment dependencies. - { - pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + // Test dependencies take the same forms as the pipeline's environment dependencies. + dependencies := []dyn.Pattern{ + base.Append(dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + } + + var patterns []pipelineRewritePattern + for _, pattern := range dependencies { + patterns = append(patterns, pipelineRewritePattern{ + pattern: pattern, mode: TranslateModeLocalRelativeWithPrefix, skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, - }, + }) } - - pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ - { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - TranslateModeEnvironmentPipFlag, - func(s string) bool { - _, _, ok := libraries.IsLocalPathInPipFlag(s) - return !ok - }, - }, - { - base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), - TranslateModeEnvironmentPipFlag, - func(s string) bool { + for _, pattern := range dependencies { + patterns = append(patterns, pipelineRewritePattern{ + pattern: pattern, + mode: TranslateModeEnvironmentPipFlag, + skipRewrite: func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) return !ok }, - }, + }) } - - return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) + return patterns } func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { From 5b54854d51f3b2e8f8094688376a61845a5247b3 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 15:54:05 +0000 Subject: [PATCH 3/5] Use the patched wheel in the test dependencies of pipelines With dynamic_version, the deploy uploaded the patched wheel, but testing.environment.dependencies kept the original file name, so tests could run cached code. Co-authored-by: Isaac --- .../testing-dynamic-version/databricks.yml | 25 +++++++++++++++++ .../testing-dynamic-version/out.test.toml | 2 ++ .../testing-dynamic-version/output.txt | 27 +++++++++++++++++++ .../pipelines/testing-dynamic-version/script | 5 ++++ .../testing-dynamic-version/test.toml | 4 +++ .../tests/test_orders.py | 2 ++ .../transformations/orders.py | 1 + bundle/libraries/switch_to_patched_wheels.go | 18 +++++++++++++ 8 files changed, 84 insertions(+) create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/script create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml new file mode 100644 index 00000000000..09b1f8647e0 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml @@ -0,0 +1,25 @@ +bundle: + name: pipeline-testing-dynamic-version + +artifacts: + other_test_code: + type: whl + files: + - source: dist/other_test_code-0.0.1-py3-none-any.whl + dynamic_version: true + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - ./dist/other_test_code-0.0.1-py3-none-any.whl + - ./dist/other_test_code-0.0.1-py3-none-any.whl[test] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt b/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt new file mode 100644 index 00000000000..9af556df861 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt @@ -0,0 +1,27 @@ + +=== Test dependencies use the patched wheel, with extras kept + +>>> [CLI] bundle deploy +Uploading .databricks/bundle/default/patched_wheels/other_test_code_other_test_code/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/files... +Created pipelines.orders +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +[ + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl[test]" +] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script new file mode 100644 index 00000000000..0b3ca1ea393 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script @@ -0,0 +1,5 @@ +title "Test dependencies use the patched wheel, with extras kept" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '.body.testing.environment.dependencies' +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml new file mode 100644 index 00000000000..25d8c36639f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml @@ -0,0 +1,4 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 3c5ff184a97..0169f1adb3a 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -95,6 +95,24 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } + // Update resources.pipelines.*.testing.environment.dependencies[*] + for pipelineName, pipelineRef := range b.Config.Resources.Pipelines { + if pipelineRef == nil || pipelineRef.Testing == nil || pipelineRef.Testing.Environment == nil { + continue + } + deps := pipelineRef.Testing.Environment.Dependencies + for depInd, dep := range deps { + base, extras := patchwheel.SplitWheelExtras(dep) + repl := replacements[base] + if repl != "" { + log.Debugf(ctx, "Updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s to %s", pipelineName, depInd, dep, repl+extras) + deps[depInd] = repl + extras + } else { + log.Debugf(ctx, "Not updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s. Available replacements: %v", pipelineName, depInd, dep, slices.Sorted(maps.Keys(replacements))) + } + } + } + return nil } From 1c74f67c9e0f45376a99e529c7cf3ac7fb1a50ec Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 15:56:36 +0000 Subject: [PATCH 4/5] Run the same-name test of pipeline testing without the variable Co-authored-by: Isaac --- .../resources/pipelines/testing-same-name-libraries/test.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml index d018f5851ee..d77028ab40a 100644 --- a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -1,6 +1,5 @@ # Runs only against the local test server. Cloud = false -Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" [[Repls]] Old = '\\' From 1706c59eb8435b0455ac602cbae106738b31734c Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 16:16:53 +0000 Subject: [PATCH 5/5] Copy the prebuilt wheel in the dynamic_version test of pipeline testing dist/ is ignored, so the test now copies the wheel from the artifacts tests. Co-authored-by: Isaac --- .../bundle/resources/pipelines/testing-dynamic-version/script | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script index 0b3ca1ea393..e71c775893f 100644 --- a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script @@ -1,3 +1,7 @@ +trap "rm -fr dist" EXIT +mkdir dist +cp $TESTDIR/../../../artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl dist + title "Test dependencies use the patched wheel, with extras kept" trace $CLI bundle deploy trace print_requests.py //api/2.0/pipelines --nostamp | jq '.body.testing.environment.dependencies'