Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
bundle:
name: pipeline-testing-dynamic-version

artifacts:
other_test_code:
type: whl
files:
- source: dist/other_test_code-0.0.1-py3-none-any.whl
dynamic_version: true

resources:
pipelines:
orders:
name: orders
libraries:
- glob:
include: ./transformations/**
testing:
sources:
- glob:
include: ./tests/**
environment:
dependencies:
- ./dist/other_test_code-0.0.1-py3-none-any.whl
- ./dist/other_test_code-0.0.1-py3-none-any.whl[test]
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@

=== Test dependencies use the patched wheel, with extras kept

>>> [CLI] bundle deploy
Uploading .databricks/bundle/default/patched_wheels/other_test_code_other_test_code/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl...
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/files...
Created pipelines.orders
Files: 4 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py //api/2.0/pipelines --nostamp
[
"/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl",
"/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl[test]"
]

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.pipelines.orders

This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
delete resources.pipelines.orders

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default

Destroy: 1 deleted
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
trap "rm -fr dist" EXIT
mkdir dist
cp $TESTDIR/../../../artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl dist

title "Test dependencies use the patched wheel, with extras kept"
trace $CLI bundle deploy
trace print_requests.py //api/2.0/pipelines --nostamp | jq '.body.testing.environment.dependencies'
trace $CLI bundle destroy --auto-approve
rm "$OUT_REQUESTS"
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Runs only against the local test server.
Cloud = false
RecordRequests = true
Ignore = [".databricks"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def test_orders():
pass
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from pyspark import pipelines as dp
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
bundle:
name: pipeline-testing-same-name-libraries

resources:
pipelines:
orders:
name: orders
libraries:
- file:
path: ./transformations/orders.py
environment:
dependencies:
- ./pipeline/test_helpers-0.1-py3-none-any.whl
testing:
sources:
- glob:
include: ./tests/**
environment:
dependencies:
# A different wheel with the same file name would be uploaded to the same path.
- ./tests/test_helpers-0.1-py3-none-any.whl
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@

>>> musterr [CLI] bundle deploy
Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl
at resources.pipelines.orders.environment.dependencies[0]
resources.pipelines.orders.testing.environment.dependencies[0]
in databricks.yml:13:13
databricks.yml:21:15

Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
trace musterr $CLI bundle deploy
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Runs only against the local test server.
Cloud = false

[[Repls]]
Old = '\\'
New = '/'
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def test_orders():
pass
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
x = 1
8 changes: 5 additions & 3 deletions acceptance/bundle/resources/pipelines/testing/output.txt
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@

=== Relative test paths resolve to the deployed workspace paths

>>> [CLI] bundle validate -o json
{
"environment": {
"dependencies": [
"-e .[test]",
"-r ./tests/requirements.txt",
"-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]",
"-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt",
"./dist/*.whl"
]
},
"sources": [
{
"glob": {
"include": "./tests/**"
"include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**"
}
}
]
Expand Down
1 change: 1 addition & 0 deletions acceptance/bundle/resources/pipelines/testing/script
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
title "Relative test paths resolve to the deployed workspace paths"
trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add a bundle deploy here (RecordRequests is already on in test.toml) so we see the test wheel uploaded and ./dist/*.whl rewritten to its .internal path? This test only runs validate, so the wheel stays ./dist/*.whl in the output, and the same-name test fails before upload, so nothing in the goldens shows the upload the description talks about. If that has to wait for #7438, worth a note in the script.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The deploy step is in #7438 (7401411), in this same script. Its golden shows the test wheel uploaded and the dependency rewritten to its .internal path. I kept this PR to validate only, so each PR shows one step.

54 changes: 24 additions & 30 deletions bundle/config/mutator/paths/pipeline_paths_visitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,48 +44,42 @@ func pipelineRewritePatterns() []pipelineRewritePattern {
mode: TranslateModeDirectory,
skipRewrite: noSkipRewrite,
},
{
pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")),
mode: TranslateModeGlob,
skipRewrite: noSkipRewrite,
},
}

return allPatterns
}

func pipelineLibrariesRewritePatterns() []pipelineRewritePattern {
pipelineEnvironmentsPatterns := []pipelineRewritePattern{
{
pattern: dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("pipelines"),
dyn.AnyKey(),
dyn.Key("environment"),
dyn.Key("dependencies"),
dyn.AnyIndex(),
),
mode: TranslateModeLocalRelativeWithPrefix,
skipRewrite: func(s string) bool {
return !libraries.IsLibraryLocal(s)
},
},
// Test dependencies take the same forms as the pipeline's environment dependencies.
dependencies := []dyn.Pattern{
base.Append(dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),
base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),
}

pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{
{
dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("pipelines"),
dyn.AnyKey(),
dyn.Key("environment"),
dyn.Key("dependencies"),
dyn.AnyIndex(),
),
TranslateModeEnvironmentPipFlag,
func(s string) bool {
var patterns []pipelineRewritePattern
for _, pattern := range dependencies {
patterns = append(patterns, pipelineRewritePattern{
pattern: pattern,
mode: TranslateModeLocalRelativeWithPrefix,
skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) },
})
}
for _, pattern := range dependencies {
patterns = append(patterns, pipelineRewritePattern{
pattern: pattern,
mode: TranslateModeEnvironmentPipFlag,
skipRewrite: func(s string) bool {
_, _, ok := libraries.IsLocalPathInPipFlag(s)
return !ok
},
},
})
}

return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...)
return patterns
}

func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) {
Expand Down
7 changes: 7 additions & 0 deletions bundle/config/mutator/paths/pipeline_paths_visitor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,13 @@ func TestVisitPipelinePaths(t *testing.T) {
"src/foo.whl",
},
},
// The acceptance test bundle/resources/pipelines/testing covers the testing paths
// that are rewritten. A package name is not a path.
Testing: &pipelines.PipelineTesting{
Environment: &pipelines.PipelineTestEnvironment{
Dependencies: []string{"faker"},
},
},
},
},
},
Expand Down
13 changes: 13 additions & 0 deletions bundle/libraries/expand_glob_references.go
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern(
dyn.Key("dependencies"),
)

var pipelineTestingEnvDepsPattern = dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("pipelines"),
dyn.AnyKey(),
dyn.Key("testing"),
dyn.Key("environment"),
dyn.Key("dependencies"),
)

var clusterLibrariesPattern = dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("clusters"),
Expand All @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
pattern: pipelineEnvDepsPattern,
fn: expandEnvironmentDeps,
},
{
pattern: pipelineTestingEnvDepsPattern,
fn: expandEnvironmentDeps,
},
{
pattern: clusterLibrariesPattern,
fn: expandLibraries,
Expand Down
1 change: 1 addition & 0 deletions bundle/libraries/remote_path.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err
clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")),
envDepsPattern.Append(dyn.AnyIndex()),
pipelineEnvDepsPattern.Append(dyn.AnyIndex()),
pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()),
// The AI Runtime task's code_source_path is a local archive (typically an
// artifact-built .tar.gz) that must be uploaded and referenced by its remote
// path, exactly like a wheel or jar library.
Expand Down
1 change: 1 addition & 0 deletions bundle/libraries/same_name_libraries.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{
clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")),
envDepsPattern.Append(dyn.AnyIndex()),
pipelineEnvDepsPattern.Append(dyn.AnyIndex()),
pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()),
}

type libData struct {
Expand Down
18 changes: 18 additions & 0 deletions bundle/libraries/switch_to_patched_wheels.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,24 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag
}
}

// Update resources.pipelines.*.testing.environment.dependencies[*]
for pipelineName, pipelineRef := range b.Config.Resources.Pipelines {
if pipelineRef == nil || pipelineRef.Testing == nil || pipelineRef.Testing.Environment == nil {
continue
}
deps := pipelineRef.Testing.Environment.Dependencies
for depInd, dep := range deps {
base, extras := patchwheel.SplitWheelExtras(dep)
repl := replacements[base]
if repl != "" {
log.Debugf(ctx, "Updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s to %s", pipelineName, depInd, dep, repl+extras)
deps[depInd] = repl + extras
} else {
log.Debugf(ctx, "Not updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s. Available replacements: %v", pipelineName, depInd, dep, slices.Sorted(maps.Keys(replacements)))
}
}
}

return nil
}

Expand Down
Loading