diff --git a/.nextchanges/bundles/bundle-generate-init-bare.md b/.nextchanges/bundles/bundle-generate-init-bare.md new file mode 100644 index 00000000000..01e9533d970 --- /dev/null +++ b/.nextchanges/bundles/bundle-generate-init-bare.md @@ -0,0 +1 @@ +* Add the `default-bare` template to `databricks bundle init` and the `--init-bare` flag to `databricks bundle generate`, which creates that bare bundle in the current directory when there is no `databricks.yml` yet. ([#7355](https://github.com/databricks/cli/pull/7355)) diff --git a/acceptance/bundle/generate/init_bare_rejected_flags/out.test.toml b/acceptance/bundle/generate/init_bare_rejected_flags/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/generate/init_bare_rejected_flags/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/generate/init_bare_rejected_flags/output.txt b/acceptance/bundle/generate/init_bare_rejected_flags/output.txt new file mode 100644 index 00000000000..7a2a42a41c5 --- /dev/null +++ b/acceptance/bundle/generate/init_bare_rejected_flags/output.txt @@ -0,0 +1,22 @@ + +>>> errcode [CLI] bundle generate job --existing-job-id 1234 +Error: no databricks.yml found in this directory or any parent; run `databricks bundle init default-bare` first or re-run with --init-bare + +Exit code: 1 + +>>> errcode [CLI] bundle generate job --existing-job-id 1234 --init-bare --bind +Error: --bind requires an existing bundle and cannot be combined with --init-bare + +Exit code: 1 + +>>> errcode [CLI] bundle generate dashboard --resource my_dashboard --init-bare +Error: --resource requires an existing bundle and cannot be combined with --init-bare + +Exit code: 1 + +>>> errcode [CLI] bundle generate genie-space --resource my_space --watch --init-bare +Error: --resource requires an existing bundle and cannot be combined with --init-bare + +Exit code: 1 + +>>> find.py databricks.yml --expect 0 diff --git a/acceptance/bundle/generate/init_bare_rejected_flags/script b/acceptance/bundle/generate/init_bare_rejected_flags/script new file mode 100644 index 00000000000..b54ff695370 --- /dev/null +++ b/acceptance/bundle/generate/init_bare_rejected_flags/script @@ -0,0 +1,9 @@ +# There is no databricks.yml here. Without --init-bare generate points at it. +trace errcode $CLI bundle generate job --existing-job-id 1234 + +# Flags that need an existing bundle are rejected before anything is written. +trace errcode $CLI bundle generate job --existing-job-id 1234 --init-bare --bind +trace errcode $CLI bundle generate dashboard --resource my_dashboard --init-bare +trace errcode $CLI bundle generate genie-space --resource my_space --watch --init-bare + +trace find.py 'databricks.yml' --expect 0 diff --git a/acceptance/bundle/generate/job_init_bare/out.test.toml b/acceptance/bundle/generate/job_init_bare/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/generate/job_init_bare/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/generate/job_init_bare/output.txt b/acceptance/bundle/generate/job_init_bare/output.txt new file mode 100644 index 00000000000..e902cc5cfb5 --- /dev/null +++ b/acceptance/bundle/generate/job_init_bare/output.txt @@ -0,0 +1,47 @@ + +>>> [CLI] bundle generate job --existing-job-id 1234 --key out --init-bare +No databricks.yml found. Initializing a bare bundle skeleton. +Welcome to the bare Declarative Automation Bundle template! + +This template creates the smallest bundle skeleton, meant for importing existing resources with `databricks bundle generate`. + +Your workspace at [DATABRICKS_URL] is used for initialization. + +✨ Your bare bundle has been created. + +Add generated resources under `resources/` and supporting code under `src/`. +File successfully saved to src/outnotebook.py +Job configuration successfully saved to resources/out.job.yml + +>>> find.py ^(resources|src)/ +resources/.gitkeep +resources/out.job.yml +src/.gitkeep +src/outnotebook.py + +>>> cat databricks.yml +# This is a Declarative Automation Bundle definition for my_project. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: my_project + uuid: [UUID] + +include: + - resources/*.yml + +targets: + dev: + default: true + mode: development + workspace: + host: [DATABRICKS_URL] + +>>> cat resources/out.job.yml +resources: + jobs: + out: + name: gitjob + tasks: + - task_key: test_task + notebook_task: + notebook_path: ../src/outnotebook.py diff --git a/acceptance/bundle/generate/job_init_bare/script b/acceptance/bundle/generate/job_init_bare/script new file mode 100644 index 00000000000..f6ca8003f32 --- /dev/null +++ b/acceptance/bundle/generate/job_init_bare/script @@ -0,0 +1,13 @@ +# The bundle takes its name from the directory it is created in. +mkdir my_project +cd my_project + +# There is no databricks.yml here; --init-bare creates the bare bundle first. +trace $CLI bundle generate job --existing-job-id 1234 --key out --init-bare + +trace find.py '^(resources|src)/' +trace cat databricks.yml +trace cat resources/out.job.yml + +cd .. +rm -r my_project diff --git a/acceptance/bundle/generate/job_init_bare/test.toml b/acceptance/bundle/generate/job_init_bare/test.toml new file mode 100644 index 00000000000..7249ecca630 --- /dev/null +++ b/acceptance/bundle/generate/job_init_bare/test.toml @@ -0,0 +1,35 @@ +[[Server]] +Pattern = "GET /api/2.2/jobs/get" +Response.Body = ''' +{ + "job_id": 11223344, + "settings": { + "name": "gitjob", + "tasks": [ + { + "task_key": "test_task", + "notebook_task": { + "notebook_path": "/Workspace/Users/tester@databricks.com/outnotebook" + } + } + ] + } +} +''' + +[[Server]] +Pattern = "GET /api/2.0/workspace/get-status" +Response.Body = ''' +{ + "path": "/Workspace/Users/tester@databricks.com/outnotebook", + "object_type": "NOTEBOOK", + "language": "PYTHON", + "repos_export_format": "SOURCE" +} +''' + +[[Server]] +Pattern = "GET /api/2.0/workspace/export" +Response.Body = ''' +print("Hello, World!") +''' diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt index 41ed2d6c02a..8267679fb8f 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/output.txt +++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt @@ -47,6 +47,7 @@ Flags: Global Flags: --debug enable debug logging + --init-bare create a bare bundle in the current directory if there is no databricks.yml yet --key string resource key to use for the generated configuration -o, --output type output type: text or json (default text) -p, --profile string ~/.databrickscfg profile diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt index 092babc9af9..0893027b164 100644 --- a/acceptance/bundle/help/bundle-generate-job/output.txt +++ b/acceptance/bundle/help/bundle-generate-job/output.txt @@ -38,6 +38,7 @@ Flags: Global Flags: --debug enable debug logging + --init-bare create a bare bundle in the current directory if there is no databricks.yml yet --key string resource key to use for the generated configuration -o, --output type output type: text or json (default text) -p, --profile string ~/.databrickscfg profile diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt index 884cd8614e8..e6449c18132 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/output.txt +++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt @@ -36,6 +36,7 @@ Flags: Global Flags: --debug enable debug logging + --init-bare create a bare bundle in the current directory if there is no databricks.yml yet --key string resource key to use for the generated configuration -o, --output type output type: text or json (default text) -p, --profile string ~/.databrickscfg profile diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 39ce9293539..b54efa3d370 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -8,6 +8,7 @@ Common patterns: databricks bundle generate dashboard --existing-path /my-dashboard --key sales_dash databricks bundle generate dashboard --resource my_dashboard --watch --force # Keep local copy in sync. Useful for development. databricks bundle generate dashboard --resource my_dashboard --force # Do a one-time sync. + databricks bundle generate job --existing-job-id 123 --init-bare # Start a new bundle from an existing job. Migration workflows: @@ -22,6 +23,7 @@ Migration workflows: Use --key to specify the resource name in your bundle configuration. Use --bind to automatically bind the generated resource to the existing workspace resource. +Use --init-bare to create a bare bundle in the current directory when there is no databricks.yml yet. Usage: databricks bundle generate [command] @@ -36,6 +38,7 @@ Available Commands: Flags: -h, --help help for generate + --init-bare create a bare bundle in the current directory if there is no databricks.yml yet --key string resource key to use for the generated configuration Global Flags: diff --git a/acceptance/bundle/help/bundle-init/output.txt b/acceptance/bundle/help/bundle-init/output.txt index f11b6b51005..9bc1065ab33 100644 --- a/acceptance/bundle/help/bundle-init/output.txt +++ b/acceptance/bundle/help/bundle-init/output.txt @@ -6,6 +6,7 @@ TEMPLATE_PATH optionally specifies which template to use. It can be one of the f - default-python: The default Python template for Notebooks and Lakeflow - default-sql: The default SQL template for .sql files that run with Databricks SQL - default-minimal: The minimal template, for advanced users +- default-bare: The bare template for importing existing resources - default-scala: The default Scala template for JAR jobs - lakeflow-integrations: The template for Lakeflow integrations - dbt-sql: The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks) diff --git a/acceptance/bundle/templates/default-bare/minimal/input.json b/acceptance/bundle/templates/default-bare/minimal/input.json new file mode 100644 index 00000000000..2b9f49a7b16 --- /dev/null +++ b/acceptance/bundle/templates/default-bare/minimal/input.json @@ -0,0 +1,3 @@ +{ + "project_name": "my_bare_bundle" +} diff --git a/acceptance/bundle/templates/default-bare/minimal/out.test.toml b/acceptance/bundle/templates/default-bare/minimal/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/templates/default-bare/minimal/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/templates/default-bare/minimal/output.txt b/acceptance/bundle/templates/default-bare/minimal/output.txt new file mode 100644 index 00000000000..2ca6c07d941 --- /dev/null +++ b/acceptance/bundle/templates/default-bare/minimal/output.txt @@ -0,0 +1,30 @@ + +>>> [CLI] bundle init default-bare --config-file ./input.json --output-dir output -o json +Welcome to the bare Declarative Automation Bundle template! + +This template creates the smallest bundle skeleton, meant for importing existing resources with `databricks bundle generate`. + +Your workspace at [DATABRICKS_URL] is used for initialization. + +✨ Your bare bundle has been created. + +Add generated resources under `resources/` and supporting code under `src/`. +{ + "outputs": [ + ".gitignore", + "README.md", + "databricks.yml", + "resources/.gitkeep", + "src/.gitkeep" + ] +} + +>>> [CLI] bundle validate -t dev +Name: my_bare_bundle +Target: dev +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_bare_bundle/dev + +Validation OK! diff --git a/acceptance/bundle/templates/default-bare/minimal/output/README.md b/acceptance/bundle/templates/default-bare/minimal/output/README.md new file mode 100644 index 00000000000..a64c058fd74 --- /dev/null +++ b/acceptance/bundle/templates/default-bare/minimal/output/README.md @@ -0,0 +1,6 @@ +# my_bare_bundle + +This is a bare Declarative Automation Bundle skeleton. + +- Put generated bundle resource definitions in `resources/` +- Put notebooks, scripts, and other source files in `src/` diff --git a/acceptance/bundle/templates/default-bare/minimal/output/databricks.yml b/acceptance/bundle/templates/default-bare/minimal/output/databricks.yml new file mode 100644 index 00000000000..81736a4fb61 --- /dev/null +++ b/acceptance/bundle/templates/default-bare/minimal/output/databricks.yml @@ -0,0 +1,15 @@ +# This is a Declarative Automation Bundle definition for my_bare_bundle. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: my_bare_bundle + uuid: [UUID] + +include: + - resources/*.yml + +targets: + dev: + default: true + mode: development + workspace: + host: [DATABRICKS_URL] diff --git a/acceptance/bundle/templates/default-bare/minimal/output/out.gitignore b/acceptance/bundle/templates/default-bare/minimal/output/out.gitignore new file mode 100644 index 00000000000..531027597b4 --- /dev/null +++ b/acceptance/bundle/templates/default-bare/minimal/output/out.gitignore @@ -0,0 +1,3 @@ +.databricks/ +.venv/ +dist/ diff --git a/acceptance/bundle/templates/default-bare/minimal/output/resources/.gitkeep b/acceptance/bundle/templates/default-bare/minimal/output/resources/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/templates/default-bare/minimal/output/src/.gitkeep b/acceptance/bundle/templates/default-bare/minimal/output/src/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/templates/default-bare/minimal/script b/acceptance/bundle/templates/default-bare/minimal/script new file mode 100644 index 00000000000..91893839242 --- /dev/null +++ b/acceptance/bundle/templates/default-bare/minimal/script @@ -0,0 +1,15 @@ +trace $CLI bundle init default-bare --config-file ./input.json --output-dir output -o json + +cd output + +# Verify that empty directories are preserved +[ -d "src" ] || exit 1 +[ -d "resources" ] || exit 1 + +trace $CLI bundle validate -t dev + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore +rm -r .databricks + +cd .. diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go index 12452293bc6..f7041c86693 100644 --- a/cmd/bundle/generate.go +++ b/cmd/bundle/generate.go @@ -7,6 +7,7 @@ import ( func newGenerateCommand() *cobra.Command { var key string + var initBare bool cmd := &cobra.Command{ Use: "generate", @@ -19,6 +20,7 @@ Common patterns: databricks bundle generate dashboard --existing-path /my-dashboard --key sales_dash databricks bundle generate dashboard --resource my_dashboard --watch --force # Keep local copy in sync. Useful for development. databricks bundle generate dashboard --resource my_dashboard --force # Do a one-time sync. + databricks bundle generate job --existing-job-id 123 --init-bare # Start a new bundle from an existing job. Migration workflows: @@ -32,7 +34,8 @@ Migration workflows: 2. Deploy: databricks bundle deploy Use --key to specify the resource name in your bundle configuration. -Use --bind to automatically bind the generated resource to the existing workspace resource.`, +Use --bind to automatically bind the generated resource to the existing workspace resource. +Use --init-bare to create a bare bundle in the current directory when there is no databricks.yml yet.`, } cmd.AddCommand(generate.NewGenerateJobCommand()) @@ -42,5 +45,6 @@ Use --bind to automatically bind the generated resource to the existing workspac cmd.AddCommand(generate.NewGenerateAppCommand()) cmd.AddCommand(generate.NewGenerateGenieSpaceCommand()) cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`) + cmd.PersistentFlags().BoolVar(&initBare, "init-bare", false, `create a bare bundle in the current directory if there is no databricks.yml yet`) return cmd } diff --git a/cmd/bundle/generate/alert.go b/cmd/bundle/generate/alert.go index cb78b24de1e..284985fd3c1 100644 --- a/cmd/bundle/generate/alert.go +++ b/cmd/bundle/generate/alert.go @@ -10,11 +10,9 @@ import ( "path/filepath" "github.com/databricks/cli/bundle/generate" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlsaver" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/sql" @@ -62,13 +60,11 @@ After generation, you can deploy this alert to other targets using: cmd.Flags().BoolVarP(&force, "force", "f", false, `force overwrite existing files in the output directory`) cmd.RunE = func(cmd *cobra.Command, args []string) error { - ctx := logdiag.InitContext(cmd.Context()) - cmd.SetContext(ctx) - - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + b, err := configureBundle(cmd) + if err != nil { + return err } + ctx := cmd.Context() w := b.WorkspaceClient(ctx) diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go index bec005408cd..61706a52b5d 100644 --- a/cmd/bundle/generate/app.go +++ b/cmd/bundle/generate/app.go @@ -6,11 +6,9 @@ import ( "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/cmd/bundle/deployment" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlsaver" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" @@ -62,13 +60,11 @@ per target environment.`, cmd.Flags().MarkHidden("bind") cmd.RunE = func(cmd *cobra.Command, args []string) error { - ctx := logdiag.InitContext(cmd.Context()) - cmd.SetContext(ctx) - - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + b, err := configureBundle(cmd, "bind") + if err != nil { + return err } + ctx := cmd.Context() w := b.WorkspaceClient(ctx) cmdio.LogString(ctx, fmt.Sprintf("Loading app '%s' configuration", appName)) diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 9c4be81b391..71c35094218 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -400,13 +400,11 @@ func (d *dashboard) runForExisting(ctx context.Context, b *bundle.Bundle) { } func (d *dashboard) RunE(cmd *cobra.Command, args []string) error { - ctx := logdiag.InitContext(cmd.Context()) - cmd.SetContext(ctx) - - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + b, err := configureBundle(cmd, "bind", "resource", "watch") + if err != nil { + return err } + ctx := cmd.Context() d.initialize(ctx, b) if logdiag.HasError(ctx) { diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 7c520abc320..730302d2012 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -318,13 +318,11 @@ func (g *genieSpace) runForExisting(ctx context.Context, b *bundle.Bundle) { } func (g *genieSpace) RunE(cmd *cobra.Command, args []string) error { - ctx := logdiag.InitContext(cmd.Context()) - cmd.SetContext(ctx) - - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + b, err := configureBundle(cmd, "bind", "resource", "watch") + if err != nil { + return err } + ctx := cmd.Context() g.initialize(ctx, b) if logdiag.HasError(ctx) { diff --git a/cmd/bundle/generate/init_bare.go b/cmd/bundle/generate/init_bare.go new file mode 100644 index 00000000000..e701276cc3d --- /dev/null +++ b/cmd/bundle/generate/init_bare.go @@ -0,0 +1,117 @@ +package generate + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/template" + "github.com/databricks/cli/libs/textutil" + "github.com/spf13/cobra" +) + +const initBareFlag = "init-bare" + +// configureBundle sets up diagnostics and loads the bundle a generate command writes into. +// +// Without a bundle it fails, unless --init-bare is set: then it materializes the +// default-bare template into the working directory and loads that. bundleOnlyFlags +// name flags that only make sense against an existing bundle (bind, resource, watch); +// they are rejected before anything is written to disk. +func configureBundle(cmd *cobra.Command, bundleOnlyFlags ...string) (*bundle.Bundle, error) { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + + b := root.TryConfigureBundle(cmd) + if logdiag.HasError(ctx) { + return nil, root.ErrAlreadyPrinted + } + if b != nil { + return b, nil + } + + initBare, err := cmd.Flags().GetBool(initBareFlag) + if err != nil { + return nil, err + } + if !initBare { + return nil, errors.New("no databricks.yml found in this directory or any parent; run `databricks bundle init default-bare` first or re-run with --init-bare") + } + for _, name := range bundleOnlyFlags { + if cmd.Flags().Changed(name) { + return nil, fmt.Errorf("--%s requires an existing bundle and cannot be combined with --%s", name, initBareFlag) + } + } + + // The template renders workspace_host from the workspace client, which the + // generators otherwise take from the bundle that does not exist yet. + err = root.MustWorkspaceClient(cmd, nil) + if err != nil { + return nil, err + } + err = initBareBundle(cmd.Context()) + if err != nil { + return nil, err + } + + b = root.MustConfigureBundle(cmd) + if b == nil || logdiag.HasError(cmd.Context()) { + return nil, root.ErrAlreadyPrinted + } + return b, nil +} + +// initBareBundle materializes the default-bare template into the working directory, +// naming the bundle after it. +func initBareBundle(ctx context.Context) error { + cwd, err := os.Getwd() + if err != nil { + return err + } + + name := textutil.NormalizeString(filepath.Base(cwd)) + if name == "" { + name = "my_bundle" + } + + configFile, err := os.CreateTemp("", "bundle-generate-init-bare-*.json") + if err != nil { + return err + } + defer os.Remove(configFile.Name()) + _, err = fmt.Fprintf(configFile, "{\"project_name\":%q}", name) + if err != nil { + configFile.Close() + return err + } + err = configFile.Close() + if err != nil { + return err + } + + cmdio.LogString(ctx, "No databricks.yml found. Initializing a bare bundle skeleton.") + + r := template.Resolver{ + TemplatePathOrUrl: string(template.DefaultBare), + ConfigFile: configFile.Name(), + OutputDir: cwd, + } + tmpl, err := r.Resolve(ctx) + if err != nil { + return err + } + defer tmpl.Reader.Cleanup(ctx) + + err = tmpl.Writer.Materialize(ctx, tmpl.Reader) + if err != nil { + return err + } + tmpl.Writer.LogTelemetry(ctx) + return nil +} diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go index db6839ade98..d80db8f858b 100644 --- a/cmd/bundle/generate/job.go +++ b/cmd/bundle/generate/job.go @@ -10,11 +10,9 @@ import ( "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/cmd/bundle/deployment" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlsaver" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/spf13/cobra" @@ -69,13 +67,11 @@ After generation, you can deploy this job to other targets using: cmd.Flags().BoolVar(&downloadSparkPythonFiles, "download-spark-python-files", false, `download workspace files referenced by spark_python_task and rewrite them to a relative path`) cmd.RunE = func(cmd *cobra.Command, args []string) error { - ctx := logdiag.InitContext(cmd.Context()) - cmd.SetContext(ctx) - - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + b, err := configureBundle(cmd, "bind") + if err != nil { + return err } + ctx := cmd.Context() w := b.WorkspaceClient(ctx) job, err := w.Jobs.Get(ctx, jobs.GetJobRequest{JobId: jobId}) diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index cdde5984d40..2cbbc7b8251 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -9,11 +9,9 @@ import ( "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/cmd/bundle/deployment" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlsaver" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" @@ -65,13 +63,11 @@ like catalogs, schemas, and compute configurations per target.`, cmd.Flags().MarkHidden("bind") cmd.RunE = func(cmd *cobra.Command, args []string) error { - ctx := logdiag.InitContext(cmd.Context()) - cmd.SetContext(ctx) - - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + b, err := configureBundle(cmd, "bind") + if err != nil { + return err } + ctx := cmd.Context() w := b.WorkspaceClient(ctx) pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineId}) diff --git a/cmd/bundle/generate_test.go b/cmd/bundle/generate_test.go new file mode 100644 index 00000000000..0cf3800e5a1 --- /dev/null +++ b/cmd/bundle/generate_test.go @@ -0,0 +1,83 @@ +package bundle + +import ( + "maps" + "slices" + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" +) + +// generateUnsupported lists the resource groups without a generate command. +// Remove a group once its command lands. A group missing from both this list and +// the command table fails the test, so every new resource type gets a decision. +var generateUnsupported = []string{ + "catalogs", + "cluster_policies", + "clusters", + "data_quality_monitors", + "database_catalogs", + "database_instances", + "experiments", + "external_locations", + "features", + "instance_pools", + "internal_immutable_snapshots", + "job_runs", + "materialization_sets", + "mcp_services", + "model_provider_services", + "model_services", + "model_serving_endpoints", + "models", + "postgres_branches", + "postgres_catalogs", + "postgres_databases", + "postgres_endpoints", + "postgres_projects", + "postgres_roles", + "postgres_snapshot_schedules", + "postgres_synced_tables", + "quality_monitors", + "registered_models", + "schemas", + "secret_scopes", + "secrets", + "sql_warehouses", + "synced_database_tables", + "vector_search_endpoints", + "vector_search_indexes", + "volumes", +} + +func TestGenerateCommandCoversAllResourceGroups(t *testing.T) { + // Generate subcommand name -> resource group it writes. + supported := map[string]string{ + "alert": "alerts", + "app": "apps", + "dashboard": "dashboards", + "genie-space": "genie_spaces", + "job": "jobs", + "pipeline": "pipelines", + } + + var names []string + for _, sub := range newGenerateCommand().Commands() { + names = append(names, sub.Name()) + } + assert.ElementsMatch(t, slices.Collect(maps.Keys(supported)), names) + + groups := config.SupportedResources() + covered := slices.Collect(maps.Values(supported)) + for _, group := range generateUnsupported { + assert.Contains(t, groups, group, "%q is not a bundle resource group; drop it from generateUnsupported", group) + assert.NotContains(t, covered, group, "%q has a generate command; drop it from generateUnsupported", group) + } + for group := range groups { + if slices.Contains(generateUnsupported, group) { + continue + } + assert.Contains(t, covered, group, "%q has no generate command; add one or list it in generateUnsupported", group) + } +} diff --git a/cmd/root/auth.go b/cmd/root/auth.go index 5c03c8cfa8c..e5fd3a9cccf 100644 --- a/cmd/root/auth.go +++ b/cmd/root/auth.go @@ -333,8 +333,13 @@ func workspaceClientOrPrompt(ctx context.Context, cfg *config.Config, allowPromp } func MustWorkspaceClient(cmd *cobra.Command, args []string) error { - ctx := logdiag.InitContext(cmd.Context()) - cmd.SetContext(ctx) + ctx := cmd.Context() + // Commands that load a bundle themselves (bundle generate --init-bare) have + // already set up logdiag by the time they fall back to auth-only mode. + if !logdiag.IsSetup(ctx) { + ctx = logdiag.InitContext(ctx) + cmd.SetContext(ctx) + } cfg := &config.Config{} liteswap.Apply(ctx, cfg) diff --git a/libs/template/template.go b/libs/template/template.go index be8e6a72cf2..265eae68a1f 100644 --- a/libs/template/template.go +++ b/libs/template/template.go @@ -26,6 +26,7 @@ type TemplateName string const ( DefaultPython TemplateName = "default-python" DefaultMinimal TemplateName = "default-minimal" + DefaultBare TemplateName = "default-bare" DefaultScala TemplateName = "default-scala" ExperimentalDefaultPython TemplateName = "experimental-default-python-vnext" DefaultSql TemplateName = "default-sql" @@ -63,6 +64,12 @@ var databricksTemplates = []Template{ Reader: &builtinReader{name: string(DefaultMinimal)}, Writer: &writerWithFullTelemetry{name: DefaultMinimal}, }, + { + name: DefaultBare, + description: "The bare template for importing existing resources", + Reader: &builtinReader{name: string(DefaultBare)}, + Writer: &writerWithFullTelemetry{name: DefaultBare}, + }, { name: DefaultScala, description: "The default Scala template for JAR jobs", diff --git a/libs/template/template_test.go b/libs/template/template_test.go index 728dd868d66..93ae3737f44 100644 --- a/libs/template/template_test.go +++ b/libs/template/template_test.go @@ -11,6 +11,7 @@ func TestTemplateHelpDescriptions(t *testing.T) { expected := `- default-python: The default Python template for Notebooks and Lakeflow - default-sql: The default SQL template for .sql files that run with Databricks SQL - default-minimal: The minimal template, for advanced users +- default-bare: The bare template for importing existing resources - default-scala: The default Scala template for JAR jobs - lakeflow-integrations: The template for Lakeflow integrations - dbt-sql: The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks) @@ -25,6 +26,7 @@ func TestTemplateOptions(t *testing.T) { {Name: "default-python", Id: "The default Python template for Notebooks and Lakeflow"}, {Name: "default-sql", Id: "The default SQL template for .sql files that run with Databricks SQL"}, {Name: "default-minimal", Id: "The minimal template, for advanced users"}, + {Name: "default-bare", Id: "The bare template for importing existing resources"}, {Name: "default-scala", Id: "The default Scala template for JAR jobs"}, {Name: "lakeflow-integrations", Id: "The template for Lakeflow integrations"}, {Name: "dbt-sql", Id: "The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)"}, @@ -62,6 +64,7 @@ func TestTemplateGetDatabricksTemplate(t *testing.T) { names := []TemplateName{ DefaultPython, DefaultMinimal, + DefaultBare, DefaultScala, DefaultSql, DbtSql, diff --git a/libs/template/templates/default-bare/databricks_template_schema.json b/libs/template/templates/default-bare/databricks_template_schema.json new file mode 100644 index 00000000000..38d6f2291ce --- /dev/null +++ b/libs/template/templates/default-bare/databricks_template_schema.json @@ -0,0 +1,22 @@ +{ + "welcome_message": "Welcome to the bare Declarative Automation Bundle template!\n\nThis template creates the smallest bundle skeleton, meant for importing existing resources with `databricks bundle generate`.\n\nYour workspace at {{workspace_host}} is used for initialization.", + "properties": { + "template_name": { + "//": "This property is always set to 'default-bare'", + "skip_prompt_if": {}, + "type": "string", + "default": "default-bare", + "description": "Template name for documentation", + "order": 0 + }, + "project_name": { + "type": "string", + "default": "my_bundle", + "description": "Bundle name", + "order": 1, + "pattern": "^[A-Za-z0-9_]+$", + "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." + } + }, + "success_message": "\n✨ Your bare bundle has been created.\n\nAdd generated resources under `resources/` and supporting code under `src/`." +} diff --git a/libs/template/templates/default-bare/template/.gitignore.tmpl b/libs/template/templates/default-bare/template/.gitignore.tmpl new file mode 100644 index 00000000000..531027597b4 --- /dev/null +++ b/libs/template/templates/default-bare/template/.gitignore.tmpl @@ -0,0 +1,3 @@ +.databricks/ +.venv/ +dist/ diff --git a/libs/template/templates/default-bare/template/README.md.tmpl b/libs/template/templates/default-bare/template/README.md.tmpl new file mode 100644 index 00000000000..9e7568eabc6 --- /dev/null +++ b/libs/template/templates/default-bare/template/README.md.tmpl @@ -0,0 +1,6 @@ +# {{.project_name}} + +This is a bare Declarative Automation Bundle skeleton. + +- Put generated bundle resource definitions in `resources/` +- Put notebooks, scripts, and other source files in `src/` diff --git a/libs/template/templates/default-bare/template/databricks.yml.tmpl b/libs/template/templates/default-bare/template/databricks.yml.tmpl new file mode 100644 index 00000000000..4d92515ad85 --- /dev/null +++ b/libs/template/templates/default-bare/template/databricks.yml.tmpl @@ -0,0 +1,15 @@ +# This is a Declarative Automation Bundle definition for {{.project_name}}. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: {{.project_name}} + uuid: {{bundle_uuid}} + +include: + - resources/*.yml + +targets: + dev: + default: true + mode: development + workspace: + host: {{workspace_host}} diff --git a/libs/template/templates/default-bare/template/resources/.gitkeep.tmpl b/libs/template/templates/default-bare/template/resources/.gitkeep.tmpl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/libs/template/templates/default-bare/template/src/.gitkeep.tmpl b/libs/template/templates/default-bare/template/src/.gitkeep.tmpl new file mode 100644 index 00000000000..e69de29bb2d