diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 2140c1e9285..b9ebb6323fa 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -23,10 +23,16 @@ example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' +The command waits for the results. It exits with code 0 only if at least one +test was collected and none failed or errored, so a run whose tests were all +skipped succeeds; otherwise it exits with code 1. --no-wait returns after the +update starts. To stop the update, run 'pipelines stop'. + Usage: databricks pipelines test [flags] [KEY] [TEST_SELECTION...] Pipeline Test Flags: + --no-wait[=true|false] Don't wait for the tests to complete. --pipeline-id string ID of the pipeline to test, instead of a KEY. --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt index d672f849190..27c6cd2712e 100644 --- a/acceptance/pipelines/test/immutable/output.txt +++ b/acceptance/pipelines/test/immutable/output.txt @@ -17,7 +17,7 @@ Error: cannot resolve test selection "tests/test_orders.py::test_backfill": rela === Absolute workspace paths and running all tests still work ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -25,7 +25,7 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure Pipeline: my_pipeline ([UUID][0]) Update: [UUID][1] ->>> [CLI] pipelines test my_pipeline +>>> [CLI] pipelines test my_pipeline --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. diff --git a/acceptance/pipelines/test/immutable/script b/acceptance/pipelines/test/immutable/script index 919649e2a1f..bf3c0b00780 100644 --- a/acceptance/pipelines/test/immutable/script +++ b/acceptance/pipelines/test/immutable/script @@ -5,6 +5,6 @@ title "Relative test paths are rejected with workspace.sync_mode immutable" trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill title "Absolute workspace paths and running all tests still work" -trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill -trace $CLI pipelines test my_pipeline +trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait +trace $CLI pipelines test my_pipeline --no-wait trace print_requests.py //updates diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 4664393739b..56c5565804d 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -1,7 +1,7 @@ === Relative test selections resolve against the pipeline root_path ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][0] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][0] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait Pipeline: [UUID][0] Update: [UUID][1] @@ -25,7 +25,7 @@ Update: [UUID][1] >>> musterr [CLI] pipelines test --pipeline-id [UUID][2] tests Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][2] /Workspace/Users/someone@example.com/tests/test_other.py::test_x +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][2] /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait Pipeline: [UUID][2] Update: [UUID][3] diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script index 3745fe70c50..bf8ae479ce7 100644 --- a/acceptance/pipelines/test/pipeline-id/script +++ b/acceptance/pipelines/test/pipeline-id/script @@ -3,12 +3,12 @@ no_root_pipeline_id=$($CLI pipelines create --json '{"name": "no-root"}' | jq -r rm "$OUT_REQUESTS" title "Relative test selections resolve against the pipeline root_path" -trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait trace print_requests.py //updates title "Without a root_path, test selections must be workspace paths" trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests -trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x +trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait trace print_requests.py //updates title "Outside a project, --pipeline-id is required" diff --git a/acceptance/pipelines/test/results/failed/out.test.toml b/acceptance/pipelines/test/results/failed/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/failed/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/failed/output.txt b/acceptance/pipelines/test/results/failed/output.txt new file mode 100644 index 00000000000..72d36c3fb8d --- /dev/null +++ b/acceptance/pipelines/test/results/failed/output.txt @@ -0,0 +1,21 @@ + +=== Failed, errored and skipped tests + +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +Pipeline: [UUID][0] +Update: [UUID][1] +Running tests... +1 of 4 PASSED tests/test_orders.py::test_valid_order ........................... [PASSED] +2 of 4 FAILED tests/test_orders.py::test_backfill .............................. [FAILED in 2.14s] +3 of 4 SKIPPED tests/test_orders.py::test_legacy_format ........................ [SKIPPED] +4 of 4 ERROR tests/test_orders.py::test_fixture ................................ [ERROR] + +Completed with 1 failed and 1 errored tests: + +Failure in tests/test_orders.py::test_backfill + AssertionError: assert 100 == 150 + +Error in tests/test_orders.py::test_fixture + RuntimeError: fixture orders setup failed + +Done. PASSED=1 FAILED=1 SKIPPED=1 ERROR=1 TOTAL=4 diff --git a/acceptance/pipelines/test/results/failed/script b/acceptance/pipelines/test/results/failed/script new file mode 100644 index 00000000000..4942d54e76f --- /dev/null +++ b/acceptance/pipelines/test/results/failed/script @@ -0,0 +1,2 @@ +title "Failed, errored and skipped tests" +trace musterr $CLI pipelines test --pipeline-id $pipeline_id diff --git a/acceptance/pipelines/test/results/failed/test.toml b/acceptance/pipelines/test/results/failed/test.toml new file mode 100644 index 00000000000..54707dfe879 --- /dev/null +++ b/acceptance/pipelines/test/results/failed/test.toml @@ -0,0 +1,25 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = '{"update_id": "00000000-0000-0000-0000-0000000000a1"}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "FAILED"}}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events" +Response.Body = ''' +{"events": [ + {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e2", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e3", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_legacy_format"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e4", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_fixture"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e5", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e6", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e7", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:03.140Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "FAILED"}}, "level": "ERROR", "error": {"fatal": false, "exceptions": [{"class_name": "AssertionError", "message": "assert 100 == 150"}]}}, + {"id": "e8", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:04.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_legacy_format"}, "details": {"test_case_progress": {"status": "SKIPPED"}}}, + {"id": "e9", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:05.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_fixture"}, "details": {"test_case_progress": {"status": "ERROR"}}, "level": "ERROR", "error": {"fatal": false, "exceptions": [{"class_name": "RuntimeError", "message": "fixture orders setup failed"}]}}, + {"id": "e10", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:06.000Z", "details": {"test_run_summary": {"total": 4, "passed": 1, "failed": 1, "skipped": 1, "errored": 1}}}, + {"id": "e11", "event_type": "update_progress", "timestamp": "2026-01-01T00:00:07.000Z", "level": "ERROR", "error": {"fatal": true, "exceptions": [{"class_name": "TestFailure", "message": "[PIPELINE_TEST_FAILED] 2 of the pipeline's tests failed (1 passed, 1 skipped)."}]}} +]} +''' diff --git a/acceptance/pipelines/test/results/incomplete/out.test.toml b/acceptance/pipelines/test/results/incomplete/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/incomplete/output.txt b/acceptance/pipelines/test/results/incomplete/output.txt new file mode 100644 index 00000000000..42c401f7ee3 --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/output.txt @@ -0,0 +1,12 @@ + +=== Compute ends before all results are received + +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +Pipeline: [UUID][0] +Update: [UUID][1] +Running tests... +1 of 2 PASSED tests/test_compute.py::test_small_order .......................... [PASSED] + +RUN INCOMPLETE: the update failed before all test results were received. +Done. PASSED=1 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 +Error: update [UUID][1] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. diff --git a/acceptance/pipelines/test/results/incomplete/script b/acceptance/pipelines/test/results/incomplete/script new file mode 100644 index 00000000000..5a3bdb6836d --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/script @@ -0,0 +1,2 @@ +title "Compute ends before all results are received" +trace musterr $CLI pipelines test --pipeline-id $pipeline_id diff --git a/acceptance/pipelines/test/results/incomplete/test.toml b/acceptance/pipelines/test/results/incomplete/test.toml new file mode 100644 index 00000000000..d78f4db704d --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/test.toml @@ -0,0 +1,19 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = '{"update_id": "00000000-0000-0000-0000-0000000000a1"}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "FAILED"}}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events" +Response.Body = ''' +{"events": [ + {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_small_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e2", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_large_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e3", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_small_order"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e4", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_large_order"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e5", "event_type": "update_progress", "timestamp": "2026-01-01T00:00:02.000Z", "level": "ERROR", "error": {"fatal": true, "exceptions": [{"class_name": "ComputeTerminatedException", "message": "Compute terminated unexpectedly."}]}} +]} +''' diff --git a/acceptance/pipelines/test/results/passed/out.test.toml b/acceptance/pipelines/test/results/passed/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/passed/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/passed/output.txt b/acceptance/pipelines/test/results/passed/output.txt new file mode 100644 index 00000000000..3abdfcd3ae6 --- /dev/null +++ b/acceptance/pipelines/test/results/passed/output.txt @@ -0,0 +1,31 @@ + +=== All tests pass + +>>> [CLI] pipelines test --pipeline-id [UUID][0] +Pipeline: [UUID][0] +Update: [UUID][1] +Running tests... +1 of 2 PASSED tests/test_orders.py::test_valid_order ........................... [PASSED in 1.84s] +2 of 2 PASSED tests/test_orders.py::test_backfill .............................. [PASSED in 1.00s] + +Done. PASSED=2 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 + +>>> print_requests.py --get //events +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID][0]/events", + "q": { + "filter": "update_id = '[UUID][1]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID][0]/events", + "q": { + "filter": "update_id = '[UUID][1]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} diff --git a/acceptance/pipelines/test/results/passed/script b/acceptance/pipelines/test/results/passed/script new file mode 100644 index 00000000000..2d87eb30ab9 --- /dev/null +++ b/acceptance/pipelines/test/results/passed/script @@ -0,0 +1,3 @@ +title "All tests pass" +trace $CLI pipelines test --pipeline-id $pipeline_id +trace print_requests.py --get //events diff --git a/acceptance/pipelines/test/results/passed/test.toml b/acceptance/pipelines/test/results/passed/test.toml new file mode 100644 index 00000000000..f821ca318eb --- /dev/null +++ b/acceptance/pipelines/test/results/passed/test.toml @@ -0,0 +1,21 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = '{"update_id": "00000000-0000-0000-0000-0000000000a1"}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "COMPLETED"}}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events" +Response.Body = ''' +{"events": [ + {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e2", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e3", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e4", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:02.840Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e5", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:03.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e6", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:04.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e7", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:05.000Z", "details": {"test_run_summary": {"total": 2, "passed": 2, "failed": 0, "skipped": 0, "errored": 0}}} +]} +''' diff --git a/acceptance/pipelines/test/results/script.prepare b/acceptance/pipelines/test/results/script.prepare new file mode 100644 index 00000000000..caaa90ba318 --- /dev/null +++ b/acceptance/pipelines/test/results/script.prepare @@ -0,0 +1,3 @@ +# Each case stubs Start Update, Get Update and List Pipeline Events in its test.toml. +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 016916e4c01..a4fab81a602 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -8,7 +8,7 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WOR === Run all tests of the only pipeline ->>> [CLI] pipelines test +>>> [CLI] pipelines test --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -27,7 +27,7 @@ Update: [UUID][1] === Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -59,7 +59,7 @@ Update: [UUID][2] === Relative paths resolve from the working directory ->>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order +>>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index 5e7efd415be..3344ef8a37d 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -2,16 +2,16 @@ trace $CLI pipelines deploy rm "$OUT_REQUESTS" title "Run all tests of the only pipeline" -trace $CLI pipelines test +trace $CLI pipelines test --no-wait trace print_requests.py //updates title "Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms" -trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast +trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast --no-wait trace print_requests.py //updates title "Relative paths resolve from the working directory" cd tests -trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order +trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order --no-wait cd .. trace print_requests.py //updates diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index 32a27c892d3..9f0dc944ea4 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -2,7 +2,7 @@ === A submit error from the service is shown as is, and no update ID is printed >>> musterr [CLI] pipelines test --pipeline-id [UUID][0] -Error: An active update '[UUID][1]' already exists for pipeline '[UUID][0]'. +Error: An active update '[UUID][1]' already exists for pipeline '[UUID][2]'. >>> print_requests.py //updates { diff --git a/acceptance/pipelines/test/submit-error/script b/acceptance/pipelines/test/submit-error/script index 0d0843028c6..405aa9d1473 100644 --- a/acceptance/pipelines/test/submit-error/script +++ b/acceptance/pipelines/test/submit-error/script @@ -1,3 +1,6 @@ +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" + title "A submit error from the service is shown as is, and no update ID is printed" -trace musterr $CLI pipelines test --pipeline-id 11111111-2222-3333-4444-555555555555 +trace musterr $CLI pipelines test --pipeline-id $pipeline_id trace print_requests.py //updates diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 66c0a23e8d0..83cb761b190 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "slices" "strings" + "time" "uuid" "github.com/databricks/cli/bundle" @@ -178,6 +179,20 @@ func rootPathSelection(rootPath, relPath string) (string, error) { return strings.TrimSuffix(rootPath, "/") + "/" + filepath.ToSlash(relPath), nil } +// relativeTestID shows a test ID whose file is under folder relative to that folder, as pytest names +// tests from the project root. Other IDs are returned unchanged. The service reports IDs with the +// /Workspace prefix, which a folder may lack, so both are compared without it. +func relativeTestID(folder, id string) string { + if folder == "" { + return id + } + prefix := strings.TrimSuffix(strings.TrimPrefix(folder, "/Workspace"), "/") + "/" + if rel, ok := strings.CutPrefix(strings.TrimPrefix(id, "/Workspace"), prefix); ok { + return rel + } + return id +} + // getRootPath reads the root_path of the deployed pipeline. func getRootPath(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) @@ -206,6 +221,23 @@ type testTarget struct { pipelineID string selections []string w *databricks.WorkspaceClient + + // projectFolder is the deployed folder that test IDs are shown relative to: the bundle's + // synchronized files, or root_path. It is "" when unknown. + projectFolder string +} + +func (t testTarget) shortTestID(id string) string { + return relativeTestID(t.projectFolder, id) +} + +// shortSelections returns the selections as shortTestID shows them. +func (t testTarget) shortSelections() []string { + var short []string + for _, s := range t.selections { + short = append(short, t.shortTestID(s)) + } + return short } func (t testTarget) printHeader(out io.Writer, updateID string) { @@ -218,7 +250,7 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { } // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. -func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { +func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string, wait bool) (testTarget, error) { if cmd.Flag("target").Value.String() == "" { cmd.SetContext(root.SkipLoadBundle(cmd.Context())) } @@ -228,8 +260,9 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) + // root_path resolves relative selections and shortens the test IDs printed while waiting. var rootPath string - if hasRelativeSelection(args) { + if wait || hasRelativeSelection(args) { var err error rootPath, err = getRootPath(ctx, w, pipelineID) if err != nil { @@ -242,7 +275,7 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) if err != nil { return testTarget{}, err } - return testTarget{pipelineID: pipelineID, selections: selections, w: w}, nil + return testTarget{pipelineID: pipelineID, selections: selections, w: w, projectFolder: rootPath}, nil } // resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. @@ -286,7 +319,97 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if err != nil { return testTarget{}, err } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx), projectFolder: filesRoot}, nil +} + +const testPollInterval = time.Second + +// waitForTestUpdate polls the update until it ends, prints each test case as it finishes, and +// returns the final state. It takes the steps of the bundle pipeline runner's wait loop +// (bundle/run/pipeline.go), which cannot be reused because it starts a regular update. +func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID, updateID string, run *testRun, printer *testProgressPrinter) (pipelines.UpdateInfoState, error) { + lastTimestamp := "" + for { + response, err := w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID) + if err != nil { + return "", err + } + if response.Update == nil { + return "", fmt.Errorf("no update found with id %s for pipeline %s", updateID, pipelineID) + } + state := response.Update.State + printer.updateState(state) + + // Reading from the last timestamp returns its events again; the run ignores duplicates. + events, err := readTestEvents(ctx, w, pipelineID, updateID, lastTimestamp) + if err != nil { + return "", err + } + if len(events) > 0 { + lastTimestamp = events[len(events)-1].Timestamp + } + addTestEvents(run, printer, events) + + switch state { + case pipelines.UpdateInfoStateCompleted, pipelines.UpdateInfoStateFailed, pipelines.UpdateInfoStateCanceled: + // Events can become readable after later ones, so read all of them once more. + events, err := readTestEvents(ctx, w, pipelineID, updateID, "") + if err != nil { + return "", err + } + addTestEvents(run, printer, events) + return state, nil + default: + } + + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(testPollInterval): + } + } +} + +// addTestEvents adds events to the run and prints each case that gets its final outcome. +func addTestEvents(run *testRun, printer *testProgressPrinter, events []testPipelineEvent) { + for _, e := range events { + if c := run.add(e); c != nil { + printer.caseFinished(c, len(run.cases)) + } + } +} + +// updateError returns the error that the update reported: its own, or else the test verdict. +func (r *testRun) updateError() string { + if r.fatalError != "" { + return r.fatalError + } + return r.verdictError +} + +// testRunError returns the error the command exits with after the update ended, or nil if at +// least one test was collected and every collected test passed or was skipped. +func testRunError(updateID string, state pipelines.UpdateInfoState, run *testRun, incompleteReason string) error { + if incompleteReason == "" { + summary := run.summaries[0] + testsFailed := summary.Total == 0 || summary.Failed > 0 || summary.Errored > 0 + if !testsFailed && state == pipelines.UpdateInfoStateCompleted && run.fatalError == "" { + return nil + } + // A test update fails when its tests fail; the printed result explains it, unless the + // update ended in another way or reports an error of its own. The service's own verdict, + // such as "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed", is not. + ended := state == pipelines.UpdateInfoStateFailed || state == pipelines.UpdateInfoStateCompleted + if testsFailed && ended && run.fatalError == "" { + return root.ErrAlreadyPrinted + } + } else if state == pipelines.UpdateInfoStateCompleted { + return root.ErrAlreadyPrinted + } + if updateError := run.updateError(); updateError != "" { + return fmt.Errorf("update %s is %s: %s", updateID, state, updateError) + } + return fmt.Errorf("update %s is %s", updateID, state) } // notPipelineKeyError explains that the first argument is not a pipeline KEY, names the bundle's @@ -328,15 +451,22 @@ Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, each followed by its value in the next --pytest-arg, for example: - databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'`, + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' + +The command waits for the results. It exits with code 0 only if at least one +test was collected and none failed or errored, so a run whose tests were all +skipped succeeds; otherwise it exits with code 1. --no-wait returns after the +update starts. To stop the update, run 'pipelines stop'.`, } var pipelineID string var pytestArgs []string + var noWait bool testGroup := cmdgroup.NewFlagGroup("Pipeline Test") testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a KEY.") testGroup.FlagSet().StringArrayVar(&pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") + testGroup.FlagSet().BoolVar(&noWait, "no-wait", false, "Don't wait for the tests to complete.") wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(testGroup) @@ -362,7 +492,7 @@ example: if err != nil { return fmt.Errorf("--pipeline-id %q is not a pipeline ID", pipelineID) } - target, err = resolveTestTargetByID(cmd, args, id.String()) + target, err = resolveTestTargetByID(cmd, args, id.String(), !noWait) if err != nil { return err } @@ -381,8 +511,23 @@ example: if err != nil { return err } - target.printHeader(cmd.OutOrStdout(), updateID) - return nil + out := cmd.OutOrStdout() + target.printHeader(out, updateID) + if noWait { + return nil + } + + run := newTestRun() + run.noTestsMessage = noTestsCollectedMessage(target.shortSelections(), pytestArgs) + printer := &testProgressPrinter{out: out, shortID: target.shortTestID} + state, err := waitForTestUpdate(ctx, target.w, target.pipelineID, updateID, run, printer) + if err != nil { + return err + } + incompleteReason := run.incompleteReason(state) + renderTestResult(out, run, incompleteReason, target.shortTestID) + renderTestSummaryLine(out, run, incompleteReason) + return testRunError(updateID, state, run, incompleteReason) } return cmd diff --git a/cmd/pipelines/test_events.go b/cmd/pipelines/test_events.go index b7f88ba9f72..3066b037e95 100644 --- a/cmd/pipelines/test_events.go +++ b/cmd/pipelines/test_events.go @@ -154,6 +154,8 @@ type testRun struct { fatalError string // verdictError is the service's verdict that tests failed, if the update reported it. verdictError string + // noTestsMessage is printed when no tests were collected. It names what was run. + noTestsMessage string } // testVerdictErrorPrefix starts the message of the error by which the service reports that tests @@ -172,7 +174,7 @@ func isTestVerdict(e *pipelines.ErrorDetail) bool { } func newTestRun() *testRun { - return &testRun{seen: map[string]bool{}, byID: map[string]*testCaseResult{}} + return &testRun{seen: map[string]bool{}, byID: map[string]*testCaseResult{}, noTestsMessage: noTestsCollectedMessage(nil, nil)} } // add records an event and returns the case it gave a final outcome to, if any. diff --git a/cmd/pipelines/test_render.go b/cmd/pipelines/test_render.go index 7d5b5c8df67..4ec4422915d 100644 --- a/cmd/pipelines/test_render.go +++ b/cmd/pipelines/test_render.go @@ -101,7 +101,7 @@ func renderTestResult(out io.Writer, run *testRun, incompleteReason string, shor case incompleteReason != "": fmt.Fprintf(out, "RUN INCOMPLETE: %s.\n", incompleteReason) case run.summaries[0].Total == 0: - fmt.Fprintln(out, "No tests were collected.") + fmt.Fprintln(out, run.noTestsMessage) } } @@ -116,3 +116,21 @@ func renderTestSummaryLine(out io.Writer, run *testRun, incompleteReason string) } fmt.Fprintf(out, "Done. PASSED=%d FAILED=%d SKIPPED=%d ERROR=%d TOTAL=%d\n", s.Passed, s.Failed, s.Skipped, s.Errored, s.Total) } + +// noTestsCollectedMessage says that no tests were collected and names what was run: the selections, +// as shown to the user, or the pipeline's test sources. It does not restate pytest's naming rules, +// because a project can change them in its pytest configuration. +func noTestsCollectedMessage(selections, pytestArgs []string) string { + from := "the pipeline's test sources" + if len(selections) > 0 { + from = strings.Join(selections, ", ") + } + switch { + case len(selections) > 0: + return fmt.Sprintf("No tests were collected from %s. Check the test selection and any --pytest-arg filters.", from) + case len(pytestArgs) > 0: + return fmt.Sprintf("No tests were collected from %s. Check the --pytest-arg filters.", from) + default: + return fmt.Sprintf("No tests were collected from %s.", from) + } +} diff --git a/cmd/pipelines/test_render_test.go b/cmd/pipelines/test_render_test.go index 874a841a719..ae3d3f80698 100644 --- a/cmd/pipelines/test_render_test.go +++ b/cmd/pipelines/test_render_test.go @@ -74,7 +74,13 @@ Done. PASSED=1 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 empty := newTestRun() empty.add(ev.summary(0, 0, 0, 0, 0)) assert.Equal(t, ` -No tests were collected. +No tests were collected from the pipeline's test sources. Done. PASSED=0 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=0 `, render(empty, "")) } + +func TestNoTestsCollectedMessage(t *testing.T) { + assert.Equal(t, "No tests were collected from the pipeline's test sources.", noTestsCollectedMessage(nil, nil)) + assert.Equal(t, "No tests were collected from the pipeline's test sources. Check the --pytest-arg filters.", noTestsCollectedMessage(nil, []string{"-k", "orders"})) + assert.Equal(t, "No tests were collected from tests/test_a.py, tests/b. Check the test selection and any --pytest-arg filters.", noTestsCollectedMessage([]string{"tests/test_a.py", "tests/b"}, nil)) +} diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index d2d49539207..ede1d81c190 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -1,14 +1,19 @@ package pipelines import ( + "bytes" "errors" "os" "path/filepath" + "strings" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/testserver" + databricks "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" @@ -126,3 +131,165 @@ func TestNotPipelineKeyError(t *testing.T) { err = notPipelineKeyError(cmd, &bundle.Bundle{}, "tests", []string{"tests"}) assert.EqualError(t, err, `"tests" is not a pipeline in this bundle, and the bundle has no pipelines`) } + +func TestTestRunError(t *testing.T) { + ev := newTestEvents(t) + tests := []struct { + name string + events func() []testPipelineEvent + state pipelines.UpdateInfoState + wantErr string + }{ + { + name: "incomplete after completion", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "PASSED", "")} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "update failed with the service's verdict although no test failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + ev.testCase("t::a", "PASSED", ""), + ev.summary(1, 1, 0, 0, 0), + decodeTestEvent(t, `{"id": "fatal", "error": {"fatal": true, "exceptions": [{"message": "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed."}]}}`), + } + }, + state: pipelines.UpdateInfoStateFailed, + wantErr: "update u1 is FAILED: [PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed.", + }, + { + name: "tests failed although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "FAILED", ""), ev.summary(1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "no tests collected although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.summary(0, 0, 0, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "all tests passed but the completed update reported a fatal error", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + ev.testCase("t::a", "PASSED", ""), + ev.summary(1, 1, 0, 0, 0), + decodeTestEvent(t, `{"id": "fatal", "error": {"fatal": true, "exceptions": [{"message": "result delivery failed"}]}}`), + } + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: "update u1 is COMPLETED: result delivery failed", + }, + { + name: "all tests were skipped", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "SKIPPED", ""), ev.summary(1, 0, 0, 1, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + }, + { + name: "canceled after tests failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "FAILED", ""), ev.summary(1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCanceled, + wantErr: "update u1 is CANCELED", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events() { + run.add(e) + } + err := testRunError("u1", tc.state, run, run.incompleteReason(tc.state)) + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.wantErr) + } + }) + } +} + +func TestRelativeTestID(t *testing.T) { + id := "/Workspace/Users/someone/orders/tests/test_a.py::test_x" + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID("/Workspace/Users/someone/orders", id)) + // root_path may lack the /Workspace prefix that the service adds to test IDs. + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID("/Users/someone/orders/", id)) + assert.Equal(t, id, relativeTestID("/Workspace/Users/someone/other", id)) + assert.Equal(t, id, relativeTestID("", id)) +} + +// The wait reads events from the last timestamp, where a later event can share that timestamp, +// and reads all events again after the update ends, where an event can appear late. +func TestWaitForTestUpdate(t *testing.T) { + event := func(id, caseID, status, timestamp string) map[string]any { + return map[string]any{ + "id": id, "event_type": "test_case_progress", "timestamp": timestamp, + "origin": map[string]any{"test_case_id": caseID}, "details": map[string]any{"test_case_progress": map[string]any{"status": status}}, + } + } + a, b := "/Workspace/p/tests/test_a.py::test_a", "/Workspace/p/tests/test_a.py::test_b" + first := []map[string]any{ + event("e1", a, "PENDING", "2026-01-01T00:00:01.000Z"), + event("e2", b, "PENDING", "2026-01-01T00:00:01.000Z"), + event("e3", a, "PASSED", "2026-01-01T00:00:02.000Z"), + } + // e4 has the timestamp of the cursor; e5 and the summary are readable only after the update ends. + tie := append(first[2:], event("e4", b, "RUNNING", "2026-01-01T00:00:02.000Z")) + late := []map[string]any{event("e5", b, "PASSED", "2026-01-01T00:00:03.000Z"), { + "id": "e6", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:04.000Z", + "details": map[string]any{"test_run_summary": map[string]any{"total": 2, "passed": 2}}, + }} + + server := testserver.New(t) + polls := 0 + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/updates/{update_id}", func(req testserver.Request) any { + polls++ + state := "RUNNING" + if polls == 2 { + state = "COMPLETED" + } + return map[string]any{"update": map[string]any{"update_id": "u1", "state": state}} + }) + var filters []string + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req testserver.Request) any { + filter := req.URL.Query().Get("filter") + filters = append(filters, filter) + switch { + case !strings.Contains(filter, "timestamp") && polls == 1: + return map[string]any{"events": first} + case strings.Contains(filter, "timestamp"): + return map[string]any{"events": tie} + default: + return map[string]any{"events": append(append(first, tie...), late...)} + } + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + var out bytes.Buffer + run := newTestRun() + printer := &testProgressPrinter{out: &out, shortID: func(id string) string { return relativeTestID("/Workspace/p", id) }} + state, err := waitForTestUpdate(t.Context(), w, "p1", "u1", run, printer) + require.NoError(t, err) + + assert.Equal(t, pipelines.UpdateInfoStateCompleted, state) + assert.Equal(t, []string{ + "update_id = 'u1'", + "update_id = 'u1' AND timestamp >= '2026-01-01T00:00:02.000Z'", + "update_id = 'u1'", + }, filters) + assert.Empty(t, run.incompleteReason(state)) + assert.Equal(t, testRunSummary{Total: 2, Passed: 2}, run.counts()) + assert.Contains(t, out.String(), "2 of 2 PASSED tests/test_a.py::test_b") +}