From da9901854aae629f7320dd89e3e2290223337ada Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 10 Nov 2026 03:51:00 +0000 Subject: [PATCH 1/3] Wait for and print the results of databricks pipelines test The command waits for the test-only update, prints each test case as it finishes, then the failure details and a summary line. It exits with code 0 only for a complete run where at least one test ran and none failed or errored, and a completed update that reports a fatal error does not pass. Test IDs are shown relative to the project folder. With the no-wait flag, the command returns after the update starts. The wait takes the steps of the bundle pipeline runner's wait loop. The acceptance tests stub the service per case, as the other pipelines tests do. Co-authored-by: Isaac --- .../pipelines/test/arguments/output.txt | 6 + .../pipelines/test/immutable/output.txt | 4 +- acceptance/pipelines/test/immutable/script | 4 +- .../pipelines/test/pipeline-id/output.txt | 4 +- acceptance/pipelines/test/pipeline-id/script | 4 +- .../test/results/failed/out.test.toml | 1 + .../pipelines/test/results/failed/output.txt | 21 +++ .../pipelines/test/results/failed/script | 2 + .../pipelines/test/results/failed/test.toml | 25 +++ .../test/results/incomplete/out.test.toml | 1 + .../test/results/incomplete/output.txt | 12 ++ .../pipelines/test/results/incomplete/script | 2 + .../test/results/incomplete/test.toml | 19 ++ .../test/results/passed/out.test.toml | 1 + .../pipelines/test/results/passed/output.txt | 31 ++++ .../pipelines/test/results/passed/script | 3 + .../pipelines/test/results/passed/test.toml | 21 +++ .../pipelines/test/results/script.prepare | 3 + .../pipelines/test/selection/output.txt | 6 +- acceptance/pipelines/test/selection/script | 6 +- .../pipelines/test/submit-error/output.txt | 2 +- acceptance/pipelines/test/submit-error/script | 5 +- cmd/pipelines/test.go | 151 +++++++++++++++- cmd/pipelines/test_test.go | 167 ++++++++++++++++++ 24 files changed, 477 insertions(+), 24 deletions(-) create mode 100644 acceptance/pipelines/test/results/failed/out.test.toml create mode 100644 acceptance/pipelines/test/results/failed/output.txt create mode 100644 acceptance/pipelines/test/results/failed/script create mode 100644 acceptance/pipelines/test/results/failed/test.toml create mode 100644 acceptance/pipelines/test/results/incomplete/out.test.toml create mode 100644 acceptance/pipelines/test/results/incomplete/output.txt create mode 100644 acceptance/pipelines/test/results/incomplete/script create mode 100644 acceptance/pipelines/test/results/incomplete/test.toml create mode 100644 acceptance/pipelines/test/results/passed/out.test.toml create mode 100644 acceptance/pipelines/test/results/passed/output.txt create mode 100644 acceptance/pipelines/test/results/passed/script create mode 100644 acceptance/pipelines/test/results/passed/test.toml create mode 100644 acceptance/pipelines/test/results/script.prepare diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 2140c1e9285..b9ebb6323fa 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -23,10 +23,16 @@ example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' +The command waits for the results. It exits with code 0 only if at least one +test was collected and none failed or errored, so a run whose tests were all +skipped succeeds; otherwise it exits with code 1. --no-wait returns after the +update starts. To stop the update, run 'pipelines stop'. + Usage: databricks pipelines test [flags] [KEY] [TEST_SELECTION...] Pipeline Test Flags: + --no-wait[=true|false] Don't wait for the tests to complete. --pipeline-id string ID of the pipeline to test, instead of a KEY. --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt index d672f849190..27c6cd2712e 100644 --- a/acceptance/pipelines/test/immutable/output.txt +++ b/acceptance/pipelines/test/immutable/output.txt @@ -17,7 +17,7 @@ Error: cannot resolve test selection "tests/test_orders.py::test_backfill": rela === Absolute workspace paths and running all tests still work ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -25,7 +25,7 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure Pipeline: my_pipeline ([UUID][0]) Update: [UUID][1] ->>> [CLI] pipelines test my_pipeline +>>> [CLI] pipelines test my_pipeline --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. diff --git a/acceptance/pipelines/test/immutable/script b/acceptance/pipelines/test/immutable/script index 919649e2a1f..bf3c0b00780 100644 --- a/acceptance/pipelines/test/immutable/script +++ b/acceptance/pipelines/test/immutable/script @@ -5,6 +5,6 @@ title "Relative test paths are rejected with workspace.sync_mode immutable" trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill title "Absolute workspace paths and running all tests still work" -trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill -trace $CLI pipelines test my_pipeline +trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait +trace $CLI pipelines test my_pipeline --no-wait trace print_requests.py //updates diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 4664393739b..56c5565804d 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -1,7 +1,7 @@ === Relative test selections resolve against the pipeline root_path ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][0] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][0] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait Pipeline: [UUID][0] Update: [UUID][1] @@ -25,7 +25,7 @@ Update: [UUID][1] >>> musterr [CLI] pipelines test --pipeline-id [UUID][2] tests Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][2] /Workspace/Users/someone@example.com/tests/test_other.py::test_x +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][2] /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait Pipeline: [UUID][2] Update: [UUID][3] diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script index 3745fe70c50..bf8ae479ce7 100644 --- a/acceptance/pipelines/test/pipeline-id/script +++ b/acceptance/pipelines/test/pipeline-id/script @@ -3,12 +3,12 @@ no_root_pipeline_id=$($CLI pipelines create --json '{"name": "no-root"}' | jq -r rm "$OUT_REQUESTS" title "Relative test selections resolve against the pipeline root_path" -trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait trace print_requests.py //updates title "Without a root_path, test selections must be workspace paths" trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests -trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x +trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait trace print_requests.py //updates title "Outside a project, --pipeline-id is required" diff --git a/acceptance/pipelines/test/results/failed/out.test.toml b/acceptance/pipelines/test/results/failed/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/failed/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/failed/output.txt b/acceptance/pipelines/test/results/failed/output.txt new file mode 100644 index 00000000000..72d36c3fb8d --- /dev/null +++ b/acceptance/pipelines/test/results/failed/output.txt @@ -0,0 +1,21 @@ + +=== Failed, errored and skipped tests + +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +Pipeline: [UUID][0] +Update: [UUID][1] +Running tests... +1 of 4 PASSED tests/test_orders.py::test_valid_order ........................... [PASSED] +2 of 4 FAILED tests/test_orders.py::test_backfill .............................. [FAILED in 2.14s] +3 of 4 SKIPPED tests/test_orders.py::test_legacy_format ........................ [SKIPPED] +4 of 4 ERROR tests/test_orders.py::test_fixture ................................ [ERROR] + +Completed with 1 failed and 1 errored tests: + +Failure in tests/test_orders.py::test_backfill + AssertionError: assert 100 == 150 + +Error in tests/test_orders.py::test_fixture + RuntimeError: fixture orders setup failed + +Done. PASSED=1 FAILED=1 SKIPPED=1 ERROR=1 TOTAL=4 diff --git a/acceptance/pipelines/test/results/failed/script b/acceptance/pipelines/test/results/failed/script new file mode 100644 index 00000000000..4942d54e76f --- /dev/null +++ b/acceptance/pipelines/test/results/failed/script @@ -0,0 +1,2 @@ +title "Failed, errored and skipped tests" +trace musterr $CLI pipelines test --pipeline-id $pipeline_id diff --git a/acceptance/pipelines/test/results/failed/test.toml b/acceptance/pipelines/test/results/failed/test.toml new file mode 100644 index 00000000000..54707dfe879 --- /dev/null +++ b/acceptance/pipelines/test/results/failed/test.toml @@ -0,0 +1,25 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = '{"update_id": "00000000-0000-0000-0000-0000000000a1"}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "FAILED"}}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events" +Response.Body = ''' +{"events": [ + {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e2", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e3", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_legacy_format"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e4", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_fixture"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e5", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e6", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e7", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:03.140Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "FAILED"}}, "level": "ERROR", "error": {"fatal": false, "exceptions": [{"class_name": "AssertionError", "message": "assert 100 == 150"}]}}, + {"id": "e8", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:04.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_legacy_format"}, "details": {"test_case_progress": {"status": "SKIPPED"}}}, + {"id": "e9", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:05.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_fixture"}, "details": {"test_case_progress": {"status": "ERROR"}}, "level": "ERROR", "error": {"fatal": false, "exceptions": [{"class_name": "RuntimeError", "message": "fixture orders setup failed"}]}}, + {"id": "e10", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:06.000Z", "details": {"test_run_summary": {"total": 4, "passed": 1, "failed": 1, "skipped": 1, "errored": 1}}}, + {"id": "e11", "event_type": "update_progress", "timestamp": "2026-01-01T00:00:07.000Z", "level": "ERROR", "error": {"fatal": true, "exceptions": [{"class_name": "TestFailure", "message": "[PIPELINE_TEST_FAILED] 2 of the pipeline's tests failed (1 passed, 1 skipped)."}]}} +]} +''' diff --git a/acceptance/pipelines/test/results/incomplete/out.test.toml b/acceptance/pipelines/test/results/incomplete/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/incomplete/output.txt b/acceptance/pipelines/test/results/incomplete/output.txt new file mode 100644 index 00000000000..42c401f7ee3 --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/output.txt @@ -0,0 +1,12 @@ + +=== Compute ends before all results are received + +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +Pipeline: [UUID][0] +Update: [UUID][1] +Running tests... +1 of 2 PASSED tests/test_compute.py::test_small_order .......................... [PASSED] + +RUN INCOMPLETE: the update failed before all test results were received. +Done. PASSED=1 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 +Error: update [UUID][1] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. diff --git a/acceptance/pipelines/test/results/incomplete/script b/acceptance/pipelines/test/results/incomplete/script new file mode 100644 index 00000000000..5a3bdb6836d --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/script @@ -0,0 +1,2 @@ +title "Compute ends before all results are received" +trace musterr $CLI pipelines test --pipeline-id $pipeline_id diff --git a/acceptance/pipelines/test/results/incomplete/test.toml b/acceptance/pipelines/test/results/incomplete/test.toml new file mode 100644 index 00000000000..d78f4db704d --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/test.toml @@ -0,0 +1,19 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = '{"update_id": "00000000-0000-0000-0000-0000000000a1"}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "FAILED"}}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events" +Response.Body = ''' +{"events": [ + {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_small_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e2", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_large_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e3", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_small_order"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e4", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_large_order"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e5", "event_type": "update_progress", "timestamp": "2026-01-01T00:00:02.000Z", "level": "ERROR", "error": {"fatal": true, "exceptions": [{"class_name": "ComputeTerminatedException", "message": "Compute terminated unexpectedly."}]}} +]} +''' diff --git a/acceptance/pipelines/test/results/passed/out.test.toml b/acceptance/pipelines/test/results/passed/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/passed/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/passed/output.txt b/acceptance/pipelines/test/results/passed/output.txt new file mode 100644 index 00000000000..3abdfcd3ae6 --- /dev/null +++ b/acceptance/pipelines/test/results/passed/output.txt @@ -0,0 +1,31 @@ + +=== All tests pass + +>>> [CLI] pipelines test --pipeline-id [UUID][0] +Pipeline: [UUID][0] +Update: [UUID][1] +Running tests... +1 of 2 PASSED tests/test_orders.py::test_valid_order ........................... [PASSED in 1.84s] +2 of 2 PASSED tests/test_orders.py::test_backfill .............................. [PASSED in 1.00s] + +Done. PASSED=2 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 + +>>> print_requests.py --get //events +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID][0]/events", + "q": { + "filter": "update_id = '[UUID][1]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID][0]/events", + "q": { + "filter": "update_id = '[UUID][1]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} diff --git a/acceptance/pipelines/test/results/passed/script b/acceptance/pipelines/test/results/passed/script new file mode 100644 index 00000000000..2d87eb30ab9 --- /dev/null +++ b/acceptance/pipelines/test/results/passed/script @@ -0,0 +1,3 @@ +title "All tests pass" +trace $CLI pipelines test --pipeline-id $pipeline_id +trace print_requests.py --get //events diff --git a/acceptance/pipelines/test/results/passed/test.toml b/acceptance/pipelines/test/results/passed/test.toml new file mode 100644 index 00000000000..f821ca318eb --- /dev/null +++ b/acceptance/pipelines/test/results/passed/test.toml @@ -0,0 +1,21 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = '{"update_id": "00000000-0000-0000-0000-0000000000a1"}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "COMPLETED"}}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events" +Response.Body = ''' +{"events": [ + {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e2", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e3", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e4", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:02.840Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e5", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:03.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e6", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:04.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e7", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:05.000Z", "details": {"test_run_summary": {"total": 2, "passed": 2, "failed": 0, "skipped": 0, "errored": 0}}} +]} +''' diff --git a/acceptance/pipelines/test/results/script.prepare b/acceptance/pipelines/test/results/script.prepare new file mode 100644 index 00000000000..caaa90ba318 --- /dev/null +++ b/acceptance/pipelines/test/results/script.prepare @@ -0,0 +1,3 @@ +# Each case stubs Start Update, Get Update and List Pipeline Events in its test.toml. +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 016916e4c01..a4fab81a602 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -8,7 +8,7 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WOR === Run all tests of the only pipeline ->>> [CLI] pipelines test +>>> [CLI] pipelines test --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -27,7 +27,7 @@ Update: [UUID][1] === Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -59,7 +59,7 @@ Update: [UUID][2] === Relative paths resolve from the working directory ->>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order +>>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index 5e7efd415be..3344ef8a37d 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -2,16 +2,16 @@ trace $CLI pipelines deploy rm "$OUT_REQUESTS" title "Run all tests of the only pipeline" -trace $CLI pipelines test +trace $CLI pipelines test --no-wait trace print_requests.py //updates title "Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms" -trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast +trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast --no-wait trace print_requests.py //updates title "Relative paths resolve from the working directory" cd tests -trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order +trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order --no-wait cd .. trace print_requests.py //updates diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index 32a27c892d3..9f0dc944ea4 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -2,7 +2,7 @@ === A submit error from the service is shown as is, and no update ID is printed >>> musterr [CLI] pipelines test --pipeline-id [UUID][0] -Error: An active update '[UUID][1]' already exists for pipeline '[UUID][0]'. +Error: An active update '[UUID][1]' already exists for pipeline '[UUID][2]'. >>> print_requests.py //updates { diff --git a/acceptance/pipelines/test/submit-error/script b/acceptance/pipelines/test/submit-error/script index 0d0843028c6..405aa9d1473 100644 --- a/acceptance/pipelines/test/submit-error/script +++ b/acceptance/pipelines/test/submit-error/script @@ -1,3 +1,6 @@ +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" + title "A submit error from the service is shown as is, and no update ID is printed" -trace musterr $CLI pipelines test --pipeline-id 11111111-2222-3333-4444-555555555555 +trace musterr $CLI pipelines test --pipeline-id $pipeline_id trace print_requests.py //updates diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 66c0a23e8d0..bf9eb8b8169 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "slices" "strings" + "time" "uuid" "github.com/databricks/cli/bundle" @@ -178,6 +179,20 @@ func rootPathSelection(rootPath, relPath string) (string, error) { return strings.TrimSuffix(rootPath, "/") + "/" + filepath.ToSlash(relPath), nil } +// relativeTestID shows a test ID whose file is under folder relative to that folder, as pytest names +// tests from the project root. Other IDs are returned unchanged. The service reports IDs with the +// /Workspace prefix, which a folder may lack, so both are compared without it. +func relativeTestID(folder, id string) string { + if folder == "" { + return id + } + prefix := strings.TrimSuffix(strings.TrimPrefix(folder, "/Workspace"), "/") + "/" + if rel, ok := strings.CutPrefix(strings.TrimPrefix(id, "/Workspace"), prefix); ok { + return rel + } + return id +} + // getRootPath reads the root_path of the deployed pipeline. func getRootPath(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) @@ -206,6 +221,14 @@ type testTarget struct { pipelineID string selections []string w *databricks.WorkspaceClient + + // projectFolder is the deployed folder that test IDs are shown relative to: the bundle's + // synchronized files, or root_path. It is "" when unknown. + projectFolder string +} + +func (t testTarget) shortTestID(id string) string { + return relativeTestID(t.projectFolder, id) } func (t testTarget) printHeader(out io.Writer, updateID string) { @@ -218,7 +241,7 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { } // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. -func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { +func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string, wait bool) (testTarget, error) { if cmd.Flag("target").Value.String() == "" { cmd.SetContext(root.SkipLoadBundle(cmd.Context())) } @@ -228,8 +251,9 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) + // root_path resolves relative selections and shortens the test IDs printed while waiting. var rootPath string - if hasRelativeSelection(args) { + if wait || hasRelativeSelection(args) { var err error rootPath, err = getRootPath(ctx, w, pipelineID) if err != nil { @@ -242,7 +266,7 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) if err != nil { return testTarget{}, err } - return testTarget{pipelineID: pipelineID, selections: selections, w: w}, nil + return testTarget{pipelineID: pipelineID, selections: selections, w: w, projectFolder: rootPath}, nil } // resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. @@ -286,7 +310,97 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if err != nil { return testTarget{}, err } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx), projectFolder: filesRoot}, nil +} + +const testPollInterval = time.Second + +// waitForTestUpdate polls the update until it ends, prints each test case as it finishes, and +// returns the final state. It takes the steps of the bundle pipeline runner's wait loop +// (bundle/run/pipeline.go), which cannot be reused because it starts a regular update. +func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID, updateID string, run *testRun, printer *testProgressPrinter) (pipelines.UpdateInfoState, error) { + lastTimestamp := "" + for { + response, err := w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID) + if err != nil { + return "", err + } + if response.Update == nil { + return "", fmt.Errorf("no update found with id %s for pipeline %s", updateID, pipelineID) + } + state := response.Update.State + printer.updateState(state) + + // Reading from the last timestamp returns its events again; the run ignores duplicates. + events, err := readTestEvents(ctx, w, pipelineID, updateID, lastTimestamp) + if err != nil { + return "", err + } + if len(events) > 0 { + lastTimestamp = events[len(events)-1].Timestamp + } + addTestEvents(run, printer, events) + + switch state { + case pipelines.UpdateInfoStateCompleted, pipelines.UpdateInfoStateFailed, pipelines.UpdateInfoStateCanceled: + // Events can become readable after later ones, so read all of them once more. + events, err := readTestEvents(ctx, w, pipelineID, updateID, "") + if err != nil { + return "", err + } + addTestEvents(run, printer, events) + return state, nil + default: + } + + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(testPollInterval): + } + } +} + +// addTestEvents adds events to the run and prints each case that gets its final outcome. +func addTestEvents(run *testRun, printer *testProgressPrinter, events []testPipelineEvent) { + for _, e := range events { + if c := run.add(e); c != nil { + printer.caseFinished(c, len(run.cases)) + } + } +} + +// updateError returns the error that the update reported: its own, or else the test verdict. +func (r *testRun) updateError() string { + if r.fatalError != "" { + return r.fatalError + } + return r.verdictError +} + +// testRunError returns the error the command exits with after the update ended, or nil if at +// least one test was collected and every collected test passed or was skipped. +func testRunError(updateID string, state pipelines.UpdateInfoState, run *testRun, incompleteReason string) error { + if incompleteReason == "" { + summary := run.summaries[0] + testsFailed := summary.Total == 0 || summary.Failed > 0 || summary.Errored > 0 + if !testsFailed && state == pipelines.UpdateInfoStateCompleted && run.fatalError == "" { + return nil + } + // A test update fails when its tests fail; the printed result explains it, unless the + // update ended in another way or reports an error of its own. The service's own verdict, + // such as "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed", is not. + ended := state == pipelines.UpdateInfoStateFailed || state == pipelines.UpdateInfoStateCompleted + if testsFailed && ended && run.fatalError == "" { + return root.ErrAlreadyPrinted + } + } else if state == pipelines.UpdateInfoStateCompleted { + return root.ErrAlreadyPrinted + } + if updateError := run.updateError(); updateError != "" { + return fmt.Errorf("update %s is %s: %s", updateID, state, updateError) + } + return fmt.Errorf("update %s is %s", updateID, state) } // notPipelineKeyError explains that the first argument is not a pipeline KEY, names the bundle's @@ -328,15 +442,22 @@ Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, each followed by its value in the next --pytest-arg, for example: - databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'`, + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' + +The command waits for the results. It exits with code 0 only if at least one +test was collected and none failed or errored, so a run whose tests were all +skipped succeeds; otherwise it exits with code 1. --no-wait returns after the +update starts. To stop the update, run 'pipelines stop'.`, } var pipelineID string var pytestArgs []string + var noWait bool testGroup := cmdgroup.NewFlagGroup("Pipeline Test") testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a KEY.") testGroup.FlagSet().StringArrayVar(&pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") + testGroup.FlagSet().BoolVar(&noWait, "no-wait", false, "Don't wait for the tests to complete.") wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(testGroup) @@ -362,7 +483,7 @@ example: if err != nil { return fmt.Errorf("--pipeline-id %q is not a pipeline ID", pipelineID) } - target, err = resolveTestTargetByID(cmd, args, id.String()) + target, err = resolveTestTargetByID(cmd, args, id.String(), !noWait) if err != nil { return err } @@ -381,8 +502,22 @@ example: if err != nil { return err } - target.printHeader(cmd.OutOrStdout(), updateID) - return nil + out := cmd.OutOrStdout() + target.printHeader(out, updateID) + if noWait { + return nil + } + + run := newTestRun() + printer := &testProgressPrinter{out: out, shortID: target.shortTestID} + state, err := waitForTestUpdate(ctx, target.w, target.pipelineID, updateID, run, printer) + if err != nil { + return err + } + incompleteReason := run.incompleteReason(state) + renderTestResult(out, run, incompleteReason, target.shortTestID) + renderTestSummaryLine(out, run, incompleteReason) + return testRunError(updateID, state, run, incompleteReason) } return cmd diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index d2d49539207..ede1d81c190 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -1,14 +1,19 @@ package pipelines import ( + "bytes" "errors" "os" "path/filepath" + "strings" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/testserver" + databricks "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" @@ -126,3 +131,165 @@ func TestNotPipelineKeyError(t *testing.T) { err = notPipelineKeyError(cmd, &bundle.Bundle{}, "tests", []string{"tests"}) assert.EqualError(t, err, `"tests" is not a pipeline in this bundle, and the bundle has no pipelines`) } + +func TestTestRunError(t *testing.T) { + ev := newTestEvents(t) + tests := []struct { + name string + events func() []testPipelineEvent + state pipelines.UpdateInfoState + wantErr string + }{ + { + name: "incomplete after completion", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "PASSED", "")} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "update failed with the service's verdict although no test failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + ev.testCase("t::a", "PASSED", ""), + ev.summary(1, 1, 0, 0, 0), + decodeTestEvent(t, `{"id": "fatal", "error": {"fatal": true, "exceptions": [{"message": "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed."}]}}`), + } + }, + state: pipelines.UpdateInfoStateFailed, + wantErr: "update u1 is FAILED: [PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed.", + }, + { + name: "tests failed although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "FAILED", ""), ev.summary(1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "no tests collected although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.summary(0, 0, 0, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "all tests passed but the completed update reported a fatal error", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + ev.testCase("t::a", "PASSED", ""), + ev.summary(1, 1, 0, 0, 0), + decodeTestEvent(t, `{"id": "fatal", "error": {"fatal": true, "exceptions": [{"message": "result delivery failed"}]}}`), + } + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: "update u1 is COMPLETED: result delivery failed", + }, + { + name: "all tests were skipped", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "SKIPPED", ""), ev.summary(1, 0, 0, 1, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + }, + { + name: "canceled after tests failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "FAILED", ""), ev.summary(1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCanceled, + wantErr: "update u1 is CANCELED", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events() { + run.add(e) + } + err := testRunError("u1", tc.state, run, run.incompleteReason(tc.state)) + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.wantErr) + } + }) + } +} + +func TestRelativeTestID(t *testing.T) { + id := "/Workspace/Users/someone/orders/tests/test_a.py::test_x" + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID("/Workspace/Users/someone/orders", id)) + // root_path may lack the /Workspace prefix that the service adds to test IDs. + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID("/Users/someone/orders/", id)) + assert.Equal(t, id, relativeTestID("/Workspace/Users/someone/other", id)) + assert.Equal(t, id, relativeTestID("", id)) +} + +// The wait reads events from the last timestamp, where a later event can share that timestamp, +// and reads all events again after the update ends, where an event can appear late. +func TestWaitForTestUpdate(t *testing.T) { + event := func(id, caseID, status, timestamp string) map[string]any { + return map[string]any{ + "id": id, "event_type": "test_case_progress", "timestamp": timestamp, + "origin": map[string]any{"test_case_id": caseID}, "details": map[string]any{"test_case_progress": map[string]any{"status": status}}, + } + } + a, b := "/Workspace/p/tests/test_a.py::test_a", "/Workspace/p/tests/test_a.py::test_b" + first := []map[string]any{ + event("e1", a, "PENDING", "2026-01-01T00:00:01.000Z"), + event("e2", b, "PENDING", "2026-01-01T00:00:01.000Z"), + event("e3", a, "PASSED", "2026-01-01T00:00:02.000Z"), + } + // e4 has the timestamp of the cursor; e5 and the summary are readable only after the update ends. + tie := append(first[2:], event("e4", b, "RUNNING", "2026-01-01T00:00:02.000Z")) + late := []map[string]any{event("e5", b, "PASSED", "2026-01-01T00:00:03.000Z"), { + "id": "e6", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:04.000Z", + "details": map[string]any{"test_run_summary": map[string]any{"total": 2, "passed": 2}}, + }} + + server := testserver.New(t) + polls := 0 + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/updates/{update_id}", func(req testserver.Request) any { + polls++ + state := "RUNNING" + if polls == 2 { + state = "COMPLETED" + } + return map[string]any{"update": map[string]any{"update_id": "u1", "state": state}} + }) + var filters []string + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req testserver.Request) any { + filter := req.URL.Query().Get("filter") + filters = append(filters, filter) + switch { + case !strings.Contains(filter, "timestamp") && polls == 1: + return map[string]any{"events": first} + case strings.Contains(filter, "timestamp"): + return map[string]any{"events": tie} + default: + return map[string]any{"events": append(append(first, tie...), late...)} + } + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + var out bytes.Buffer + run := newTestRun() + printer := &testProgressPrinter{out: &out, shortID: func(id string) string { return relativeTestID("/Workspace/p", id) }} + state, err := waitForTestUpdate(t.Context(), w, "p1", "u1", run, printer) + require.NoError(t, err) + + assert.Equal(t, pipelines.UpdateInfoStateCompleted, state) + assert.Equal(t, []string{ + "update_id = 'u1'", + "update_id = 'u1' AND timestamp >= '2026-01-01T00:00:02.000Z'", + "update_id = 'u1'", + }, filters) + assert.Empty(t, run.incompleteReason(state)) + assert.Equal(t, testRunSummary{Total: 2, Passed: 2}, run.counts()) + assert.Contains(t, out.String(), "2 of 2 PASSED tests/test_a.py::test_b") +} From cfe688b0d40a2ff912d1a50cd35b1318cc6a40e1 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 16 Nov 2026 23:44:00 +0000 Subject: [PATCH 2/3] Name what was run when databricks pipelines test collects no tests The message names the test selection, or the pipeline's test sources, and points to the --pytest-arg filters when they were given. It does not restate pytest's naming rules, because a project can change them. The changelog entry for the results output moves to this PR. Co-authored-by: Isaac --- .nextchanges/cli/pipelines-test-results.md | 1 + cmd/pipelines/test.go | 10 ++++++++++ cmd/pipelines/test_events.go | 4 +++- cmd/pipelines/test_render.go | 20 +++++++++++++++++++- cmd/pipelines/test_render_test.go | 8 +++++++- 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 .nextchanges/cli/pipelines-test-results.md diff --git a/.nextchanges/cli/pipelines-test-results.md b/.nextchanges/cli/pipelines-test-results.md new file mode 100644 index 00000000000..c06ea5d3b17 --- /dev/null +++ b/.nextchanges/cli/pipelines-test-results.md @@ -0,0 +1 @@ +* `databricks pipelines test` waits for the test run and prints the result of each test and a summary. `--no-wait` returns after the run starts. ([#7432](https://github.com/databricks/cli/pull/7432), [#7433](https://github.com/databricks/cli/pull/7433), [#7298](https://github.com/databricks/cli/pull/7298)) diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index bf9eb8b8169..83cb761b190 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -231,6 +231,15 @@ func (t testTarget) shortTestID(id string) string { return relativeTestID(t.projectFolder, id) } +// shortSelections returns the selections as shortTestID shows them. +func (t testTarget) shortSelections() []string { + var short []string + for _, s := range t.selections { + short = append(short, t.shortTestID(s)) + } + return short +} + func (t testTarget) printHeader(out io.Writer, updateID string) { if t.key != "" { fmt.Fprintf(out, "Pipeline: %s (%s)\n", t.key, t.pipelineID) @@ -509,6 +518,7 @@ update starts. To stop the update, run 'pipelines stop'.`, } run := newTestRun() + run.noTestsMessage = noTestsCollectedMessage(target.shortSelections(), pytestArgs) printer := &testProgressPrinter{out: out, shortID: target.shortTestID} state, err := waitForTestUpdate(ctx, target.w, target.pipelineID, updateID, run, printer) if err != nil { diff --git a/cmd/pipelines/test_events.go b/cmd/pipelines/test_events.go index b7f88ba9f72..3066b037e95 100644 --- a/cmd/pipelines/test_events.go +++ b/cmd/pipelines/test_events.go @@ -154,6 +154,8 @@ type testRun struct { fatalError string // verdictError is the service's verdict that tests failed, if the update reported it. verdictError string + // noTestsMessage is printed when no tests were collected. It names what was run. + noTestsMessage string } // testVerdictErrorPrefix starts the message of the error by which the service reports that tests @@ -172,7 +174,7 @@ func isTestVerdict(e *pipelines.ErrorDetail) bool { } func newTestRun() *testRun { - return &testRun{seen: map[string]bool{}, byID: map[string]*testCaseResult{}} + return &testRun{seen: map[string]bool{}, byID: map[string]*testCaseResult{}, noTestsMessage: noTestsCollectedMessage(nil, nil)} } // add records an event and returns the case it gave a final outcome to, if any. diff --git a/cmd/pipelines/test_render.go b/cmd/pipelines/test_render.go index 7d5b5c8df67..4ec4422915d 100644 --- a/cmd/pipelines/test_render.go +++ b/cmd/pipelines/test_render.go @@ -101,7 +101,7 @@ func renderTestResult(out io.Writer, run *testRun, incompleteReason string, shor case incompleteReason != "": fmt.Fprintf(out, "RUN INCOMPLETE: %s.\n", incompleteReason) case run.summaries[0].Total == 0: - fmt.Fprintln(out, "No tests were collected.") + fmt.Fprintln(out, run.noTestsMessage) } } @@ -116,3 +116,21 @@ func renderTestSummaryLine(out io.Writer, run *testRun, incompleteReason string) } fmt.Fprintf(out, "Done. PASSED=%d FAILED=%d SKIPPED=%d ERROR=%d TOTAL=%d\n", s.Passed, s.Failed, s.Skipped, s.Errored, s.Total) } + +// noTestsCollectedMessage says that no tests were collected and names what was run: the selections, +// as shown to the user, or the pipeline's test sources. It does not restate pytest's naming rules, +// because a project can change them in its pytest configuration. +func noTestsCollectedMessage(selections, pytestArgs []string) string { + from := "the pipeline's test sources" + if len(selections) > 0 { + from = strings.Join(selections, ", ") + } + switch { + case len(selections) > 0: + return fmt.Sprintf("No tests were collected from %s. Check the test selection and any --pytest-arg filters.", from) + case len(pytestArgs) > 0: + return fmt.Sprintf("No tests were collected from %s. Check the --pytest-arg filters.", from) + default: + return fmt.Sprintf("No tests were collected from %s.", from) + } +} diff --git a/cmd/pipelines/test_render_test.go b/cmd/pipelines/test_render_test.go index 874a841a719..ae3d3f80698 100644 --- a/cmd/pipelines/test_render_test.go +++ b/cmd/pipelines/test_render_test.go @@ -74,7 +74,13 @@ Done. PASSED=1 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 empty := newTestRun() empty.add(ev.summary(0, 0, 0, 0, 0)) assert.Equal(t, ` -No tests were collected. +No tests were collected from the pipeline's test sources. Done. PASSED=0 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=0 `, render(empty, "")) } + +func TestNoTestsCollectedMessage(t *testing.T) { + assert.Equal(t, "No tests were collected from the pipeline's test sources.", noTestsCollectedMessage(nil, nil)) + assert.Equal(t, "No tests were collected from the pipeline's test sources. Check the --pytest-arg filters.", noTestsCollectedMessage(nil, []string{"-k", "orders"})) + assert.Equal(t, "No tests were collected from tests/test_a.py, tests/b. Check the test selection and any --pytest-arg filters.", noTestsCollectedMessage([]string{"tests/test_a.py", "tests/b"}, nil)) +} From 6dd223360cf1510fe9682ead782aacc5b47abb74 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 08:16:40 +0000 Subject: [PATCH 3/3] Move the changelog entry to the changelog PR The entry is added with the other pipeline testing entries in #7439. Co-authored-by: Isaac --- .nextchanges/cli/pipelines-test-results.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .nextchanges/cli/pipelines-test-results.md diff --git a/.nextchanges/cli/pipelines-test-results.md b/.nextchanges/cli/pipelines-test-results.md deleted file mode 100644 index c06ea5d3b17..00000000000 --- a/.nextchanges/cli/pipelines-test-results.md +++ /dev/null @@ -1 +0,0 @@ -* `databricks pipelines test` waits for the test run and prints the result of each test and a summary. `--no-wait` returns after the run starts. ([#7432](https://github.com/databricks/cli/pull/7432), [#7433](https://github.com/databricks/cli/pull/7433), [#7298](https://github.com/databricks/cli/pull/7298))