Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion acceptance/pipelines/databricks-cli-help/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ Main Commands
run Run a pipeline
show Preview a table's columns and sample rows
stop Stop a pipeline
test *Experimental* Run the tests of a pipeline
test Run the tests of a pipeline

Management Commands
apply-environment *Public Preview* Apply the latest environment to the pipeline.
Expand Down
8 changes: 0 additions & 8 deletions acceptance/pipelines/test/arguments/out.gate-off.xml

This file was deleted.

12 changes: 0 additions & 12 deletions acceptance/pipelines/test/arguments/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
>>> [CLI] pipelines test --help
Run the tests of a pipeline in a test-only update.

This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it.

KEY is the unique name of the pipeline, as defined in its YAML file.
It is required when test selections are given. If there is only one pipeline
in the project and no arguments are given, it is selected automatically.
Expand Down Expand Up @@ -58,16 +56,6 @@ Global Flags:
-t, --target string bundle target to use (if applicable)
--var strings set values for variables defined in project config. Example: --var="foo=bar"

=== The command is experimental and off by default

>>> musterr [CLI] pipelines test --pipeline-id 1234
Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it

=== With the command off, an older report is still replaced

>>> musterr [CLI] pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml
Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it

=== pytest arguments must be passed with --pytest-arg

>>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders
Expand Down
7 changes: 0 additions & 7 deletions acceptance/pipelines/test/arguments/script
Original file line number Diff line number Diff line change
@@ -1,12 +1,5 @@
trace $CLI pipelines test --help

title "The command is experimental and off by default"
(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234)

title "With the command off, an older report is still replaced"
echo '<testsuites><testsuite name="old" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.gate-off.xml
(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml)

title "pytest arguments must be passed with --pytest-arg"
trace musterr $CLI pipelines test --pipeline-id 1234 -k orders
trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders
Expand Down
3 changes: 2 additions & 1 deletion acceptance/pipelines/test/immutable/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ Update: [UUID][2]
"method": "POST",
"path": "/api/2.0/pipelines/[UUID][0]/updates",
"body": {
"test_only": true
"test_only": true,
"test_options": {}
}
}
8 changes: 8 additions & 0 deletions acceptance/pipelines/test/not-enabled/out.not-enabled.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="[UUID]" tests="1" failures="0" errors="1" skipped="0">
<testcase name="pipelines test">
<error message="Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview.">Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview.</error>
</testcase>
</testsuite>
</testsuites>
1 change: 1 addition & 0 deletions acceptance/pipelines/test/not-enabled/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
15 changes: 15 additions & 0 deletions acceptance/pipelines/test/not-enabled/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@

=== Where the testing API is off, a run of all tests is rejected, not run as an older test-only update

>>> musterr [CLI] pipelines test --pipeline-id [UUID] --junit-xml out.not-enabled.xml
Error: Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview.

>>> print_requests.py //updates
{
"method": "POST",
"path": "/api/2.0/pipelines/[UUID]/updates",
"body": {
"test_only": true,
"test_options": {}
}
}
6 changes: 6 additions & 0 deletions acceptance/pipelines/test/not-enabled/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id)
rm "$OUT_REQUESTS"

title "Where the testing API is off, a run of all tests is rejected, not run as an older test-only update"
trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.not-enabled.xml
trace print_requests.py //updates
11 changes: 11 additions & 0 deletions acceptance/pipelines/test/not-enabled/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# The service rejects a request with test_options where the testing API is off. The test server
# does not model that per-workspace setting, so the response is stubbed.
[[Server]]
Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates"
Response.Body = '''
{
"error_code": "INVALID_PARAMETER_VALUE",
"message": "Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview."
}
'''
Response.StatusCode = 400
3 changes: 2 additions & 1 deletion acceptance/pipelines/test/selection/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ Update: [UUID][1]
"method": "POST",
"path": "/api/2.0/pipelines/[UUID][0]/updates",
"body": {
"test_only": true
"test_only": true,
"test_options": {}
}
}

Expand Down
3 changes: 2 additions & 1 deletion acceptance/pipelines/test/submit-error/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ Wait for it to finish, or pass its update ID to --resume to read its results
"method": "POST",
"path": "/api/2.0/pipelines/[UUID][0]/updates",
"body": {
"test_only": true
"test_only": true,
"test_options": {}
}
}
2 changes: 0 additions & 2 deletions acceptance/pipelines/test/test.toml
Original file line number Diff line number Diff line change
@@ -1,4 +1,2 @@
RecordRequests = true
Ignore = ['.databricks']
# The command is experimental; the gate case in arguments turns it off.
Env.DATABRICKS_CLI_PIPELINES_TEST = "true"
22 changes: 7 additions & 15 deletions cmd/pipelines/test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/cmdgroup"
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/libs/logdiag"
"github.com/databricks/cli/libs/shellquote"
Expand All @@ -36,18 +35,16 @@ import (
"github.com/spf13/pflag"
)

// newStartTestUpdateRequest returns the Start Update request of a test run.
// newStartTestUpdateRequest returns the Start Update request of a test run. It always sends
// test_options, empty to run every test: only a request with test_options is rejected where the
// testing API is off; without them, the service runs its older test-only update instead.
func newStartTestUpdateRequest(selection, pytestArgs []string) pipelines.StartUpdate {
req := pipelines.StartUpdate{TestOnly: true}
if len(selection) > 0 || len(pytestArgs) > 0 {
req.TestOptions = &pipelines.PipelineTestOptions{TestSelection: selection, PytestArgs: pytestArgs}
return pipelines.StartUpdate{
TestOnly: true,
TestOptions: &pipelines.PipelineTestOptions{TestSelection: selection, PytestArgs: pytestArgs},
}
return req
}

// pipelinesTestEnvVar enables the experimental pipelines test command.
const pipelinesTestEnvVar = "DATABRICKS_CLI_PIPELINES_TEST"

const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'"

// validatePytestArgs accepts only -k and -m, each followed by its value in the next argument, as the API
Expand Down Expand Up @@ -542,9 +539,6 @@ type testFlags struct {

// checkTestFlags rejects the flag combinations that pipelines test does not support, before any API call.
func checkTestFlags(cmd *cobra.Command, args []string, f testFlags) error {
if enabled, _ := env.GetBool(cmd.Context(), pipelinesTestEnvVar); !enabled {
return fmt.Errorf("pipelines test is experimental. Set %s=true to enable it", pipelinesTestEnvVar)
}
if cmd.ArgsLenAtDash() >= 0 {
return errors.New("the -- separator is not supported: " + pytestArgHint)
}
Expand Down Expand Up @@ -614,11 +608,9 @@ func notPipelineKeyError(cmd *cobra.Command, b *bundle.Bundle, key string, args
func testCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "test [flags] [KEY] [TEST_SELECTION...]",
Short: "*Experimental* Run the tests of a pipeline",
Short: "Run the tests of a pipeline",
Long: `Run the tests of a pipeline in a test-only update.

This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it.

KEY is the unique name of the pipeline, as defined in its YAML file.
It is required when test selections are given. If there is only one pipeline
in the project and no arguments are given, it is selected automatically.
Expand Down
8 changes: 7 additions & 1 deletion cmd/pipelines/test_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package pipelines
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
Expand Down Expand Up @@ -81,7 +82,12 @@ func TestResolveTestSelections(t *testing.T) {
}

func TestNewStartTestUpdateRequest(t *testing.T) {
assert.Equal(t, pipelines.StartUpdate{TestOnly: true}, newStartTestUpdateRequest(nil, nil))
all := newStartTestUpdateRequest(nil, nil)
assert.Equal(t, pipelines.StartUpdate{TestOnly: true, TestOptions: &pipelines.PipelineTestOptions{}}, all)
// An empty test_options must reach the service, where it is rejected if the testing API is off.
body, err := json.Marshal(all)
require.NoError(t, err)
assert.JSONEq(t, `{"test_only": true, "test_options": {}}`, string(body))
assert.Equal(t, pipelines.StartUpdate{
TestOnly: true,
TestOptions: &pipelines.PipelineTestOptions{TestSelection: []string{"/Workspace/tests"}, PytestArgs: []string{"-k", "orders"}},
Expand Down
Loading