Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 164 additions & 0 deletions cmd/pipelines/test_junit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
package pipelines

import (
"encoding/xml"
"errors"
"fmt"
"io"
"path/filepath"
"strings"

"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/atomicfile"
)

type junitTestSuites struct {
XMLName xml.Name `xml:"testsuites"`
Suites []junitTestSuite `xml:"testsuite"`
}

type junitTestSuite struct {
Name string `xml:"name,attr"`
Tests int `xml:"tests,attr"`
Failures int `xml:"failures,attr"`
Errors int `xml:"errors,attr"`
Skipped int `xml:"skipped,attr"`
Cases []junitTestCase `xml:"testcase"`
}

type junitTestCase struct {
ClassName string `xml:"classname,attr,omitempty"`
Name string `xml:"name,attr"`
Time string `xml:"time,attr,omitempty"`
Failure *junitResult `xml:"failure"`
Error *junitResult `xml:"error"`
Skipped *junitResult `xml:"skipped"`
}

type junitResult struct {
Message string `xml:"message,attr,omitempty"`
Text string `xml:",chardata"`
}

// junitCaseName splits a pytest test ID the way pytest's JUnit report does: the classname
// is the module path plus any class names, joined by dots. See
// https://docs.pytest.org/en/stable/how-to/output.html#creating-junitxml-format-files.
// A parameter ID in brackets, such as test_cast[a::int], stays with the test name. Brackets are
// looked for only after the file path, which can contain them too.
func junitCaseName(testID string) (string, string) {
file, node, hasNode := strings.Cut(testID, "::")
module := strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(file, "/"), ".py"), "/", ".")
if !hasNode {
return "", module
}
params := ""
if i := strings.Index(node, "["); i >= 0 {
node, params = node[:i], node[i:]
}
parts := strings.Split(node, "::")
parts[len(parts)-1] += params
classes := append([]string{module}, parts[:len(parts)-1]...)
return strings.Join(classes, "."), parts[len(parts)-1]
}

func newJUnitResult(text string) *junitResult {
message, _, _ := strings.Cut(text, "\n")
return &junitResult{Message: message, Text: text}
}

// junitCommandEntryName names the failing entry that the command adds when the run itself has a
// problem, before or after the update starts, so that CI shows every such entry under one name.
const junitCommandEntryName = "pipelines test"

// junitNoResult is the error message of a collected case that has no final outcome.
const junitNoResult = "no result received"

// newJUnitReport has one entry per collected case; a case without a final outcome is an error,
// because CI tools treat skipped entries as passing.
// A non-empty problem, such as an incomplete run, adds a failing entry for the update so the
// report never passes by omission.
// Cases are named relative to projectFolder, as pytest names them, so that the names do not depend on
// the working directory.
func newJUnitReport(suiteName, updateID, projectFolder string, run *testRun, problem string) junitTestSuites {
suite := junitTestSuite{Name: suiteName}
for _, c := range run.cases {
entry := junitTestCase{}
entry.ClassName, entry.Name = junitCaseName(relativeTestID(projectFolder, c.id))
if d, ok := c.duration(); ok {
entry.Time = fmt.Sprintf("%.3f", d.Seconds())
}

details := ""
if c.errorDetail != nil {
details = errorDetailText(c.errorDetail)
}
if c.detailsTruncated {
details = strings.TrimPrefix(details+"\n(details truncated)", "\n")
}
switch c.status {
case testCasePassed:
case testCaseFailed:
entry.Failure = newJUnitResult(details)
suite.Failures++
case testCaseError:
entry.Error = newJUnitResult(details)
suite.Errors++
case testCaseSkipped:
entry.Skipped = &junitResult{}
suite.Skipped++
default:
entry.Error = &junitResult{Message: junitNoResult}
suite.Errors++
}
suite.Cases = append(suite.Cases, entry)
}

if problem != "" {
suite.Cases = append(suite.Cases, junitTestCase{
Name: junitCommandEntryName,
Error: newJUnitResult(problem + "\nUpdate: " + updateID),
})
suite.Errors++
}
suite.Tests = len(suite.Cases)
return junitTestSuites{Suites: []junitTestSuite{suite}}
}

// testReport writes the JUnit report of a test run to path, if --junit-xml is set.
type testReport struct {
path string
out io.Writer
suiteName string
projectFolder string
}

// finish writes the report of a run that ended with runErr. A write error is joined with runErr,
// so that neither is lost.
func (r *testReport) finish(updateID string, run *testRun, problem string, runErr error) error {
if r.path == "" {
return runErr
}
if err := writeJUnitReport(r.path, newJUnitReport(r.suiteName, updateID, r.projectFolder, run, problem)); err != nil {
if runErr == nil || errors.Is(runErr, root.ErrAlreadyPrinted) {
return err
}
return errors.Join(runErr, err)
}
fmt.Fprintf(r.out, "JUnit report: %s\n", filepath.ToSlash(r.path))
return runErr
}

// writeJUnitReport replaces the file at path with the report.
func writeJUnitReport(path string, report junitTestSuites) error {
data, err := xml.MarshalIndent(report, "", " ")
if err != nil {
return err
}
data = append([]byte(xml.Header), data...)
data = append(data, '\n')
// A partly written file would be read as a broken report, so the file is replaced at once.
if err := atomicfile.Write(path, data, 0o644); err != nil {
return fmt.Errorf("failed to write JUnit report: %w", err)
}
return nil
}
85 changes: 85 additions & 0 deletions cmd/pipelines/test_junit_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package pipelines

import (
"errors"
"io"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestJUnitCaseName(t *testing.T) {
tests := []struct {
testID, className, name string
}{
{"tests/test_orders.py::test_backfill", "tests.test_orders", "test_backfill"},
{"tests/test_orders.py::TestOrders::test_tax[a-b]", "tests.test_orders.TestOrders", "test_tax[a-b]"},
// A parameter ID may contain "::".
{"tests/test_a.py::test_cast[a::int]", "tests.test_a", "test_cast[a::int]"},
{"tests/test_a.py::TestCast::test_cast[a::int]", "tests.test_a.TestCast", "test_cast[a::int]"},
{"tests/test_orders.py", "", "tests.test_orders"},
{"/Workspace/Users/someone@example.com/tests/test_x.py::test_x", "Workspace.Users.someone@example.com.tests.test_x", "test_x"},
// A bracket in a folder name is part of the path, not a parameter ID.
{"tests/[weird]/test_a.py::test_x", "tests.[weird].test_a", "test_x"},
}
for _, tc := range tests {
className, name := junitCaseName(tc.testID)
assert.Equal(t, tc.className, className, tc.testID)
assert.Equal(t, tc.name, name, tc.testID)
}
}

func TestWriteJUnitReport(t *testing.T) {
ev := newTestEvents(t)
// Acceptance tests cover each outcome. This covers what the test server does not send:
// multi-line and truncated details, and text that XML must escape.
const project = "/Workspace/Users/someone@example.com/.bundle/orders/dev/files"
run := newTestRun()
for _, e := range []testPipelineEvent{
ev.testCase(project+"/tests/test_a.py::test_bad", "FAILED", "",
`"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2\nwhere 1 = f()"}]}`,
`"truncation": {"truncated_fields": [{"field_name": "error"}]}`),
ev.testCase(project+"/tests/test_a.py::test_setup", "ERROR", "",
`"error": {"exceptions": [{"class_name": "RuntimeError", "message": "fixture <db> failed"}]}`),
} {
run.add(e)
}

path := filepath.Join(t.TempDir(), "report.xml")
require.NoError(t, writeJUnitReport(path, newJUnitReport("orders", "u1", project, run, "")))

data, err := os.ReadFile(path)
require.NoError(t, err)
assert.Equal(t, `<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="orders" tests="2" failures="1" errors="1" skipped="0">
<testcase classname="tests.test_a" name="test_bad">
<failure message="AssertionError: assert 1 == 2">AssertionError: assert 1 == 2&#xA;where 1 = f()&#xA;(details truncated)</failure>
</testcase>
<testcase classname="tests.test_a" name="test_setup">
<error message="RuntimeError: fixture &lt;db&gt; failed">RuntimeError: fixture &lt;db&gt; failed</error>
</testcase>
</testsuite>
</testsuites>
`, string(data))
}

func TestTestReportJoinsWriteErrors(t *testing.T) {
report := &testReport{path: filepath.Join(t.TempDir(), "missing", "report.xml"), out: io.Discard, suiteName: "orders"}
runErr := errors.New("update u1 is FAILED")
err := report.finish("u1", newTestRun(), runErr.Error(), runErr)
require.ErrorIs(t, err, runErr)
assert.ErrorContains(t, err, "failed to write JUnit report")
}

func TestJUnitReportProblemEntry(t *testing.T) {
report := newJUnitReport("orders", "u1", "", newTestRun(), "RUN INCOMPLETE: the update was canceled.")
cases := report.Suites[0].Cases
require.Len(t, cases, 1)
assert.Equal(t, junitCommandEntryName, cases[0].Name)
assert.Equal(t, "RUN INCOMPLETE: the update was canceled.", cases[0].Error.Message)
assert.Equal(t, "RUN INCOMPLETE: the update was canceled.\nUpdate: u1", cases[0].Error.Text)
}
Loading