From f5bed9477801028f5055e154ebc0699e21b82cb4 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:41:27 +0000 Subject: [PATCH 1/4] [bug fix] aitools: validate skill names and file paths in the fetched skills manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `databricks aitools install` and `update` join every skill name and file path from the fetched skills manifest onto a local directory. Before this change none of them were checked, so a manifest entry could write anywhere the user can (for example `~/.bashrc`) and the command still reported success. `FetchManifest` now validates the manifest before any install or update work starts: ```diff GitHubManifestSource.FetchManifest read + json.Unmarshal manifest + validateManifestPaths(manifest) + skill name: Agent Skills name format, at most 64 chars ^[a-z0-9]+(-[a-z0-9]+)*$ + ("../x" "a/b" "." ".state.json" "Foo" "foo." "foo " rejected) + skill name: not a Windows device name, on every OS (con prn aux nul com1-9 lpt1-9) + each file: filepath.IsLocal ("/etc/x" "../../.bashrc" "a/../../x" rejected) normalizeManifest ``` Why the name rule is stricter than "a local path": - Skill directories sit next to the installer's `.state.json`. A skill with that name would replace the state file with a directory. - On case-insensitive filesystems, and through Windows trailing dots and spaces, two different names can resolve to the same directory. Only lowercase letters, digits and single hyphens are allowed, so that can't happen. - On Windows, device names like `nul` resolve to a device in any directory. They are rejected on every OS, so a given manifest passes or fails the same way on every platform. A failure returns a `FetchError` with stage `manifest`, e.g. `invalid skills manifest: skill "databricks-core" lists file "../../../.bashrc" outside its directory`. The manifest comes from `raw.githubusercontent.com/databricks/databricks-agent-skills//manifest.json` (or `DATABRICKS_SKILLS_BASE_URL`). Exploiting this needs control of what is served there, so the change is defense in depth for paths that arrive over the network, not a fix for a remote hole on its own. ## Review order 1. `libs/aitools/installer/source.go`: `validateManifestPaths`, the two name patterns, and the call in `FetchManifest`. 2. `libs/aitools/installer/source_test.go`: - `TestFetchManifestRejectsPathsOutsideSkillDir`: table of accepted and rejected names and files. - `TestInstallRejectsStateFileSkillNameWithoutTouchingState`: install, then install again from a manifest naming `.state.json`. Asserts that the install fails and the state file is byte-for-byte unchanged. ## Evidence - **Before** (the new tests run against `main`'s `source.go`): 18 of the 25 `TestFetchManifestRejectsPathsOutsideSkillDir` cases fail (every case that should be rejected) and so does `TestInstallRejectsStateFileSkillNameWithoutTouchingState`. A manifest listing `../../../../escaped-file.txt` makes `InstallSkillsForAgents` return `nil` and write `$HOME/escaped-file.txt`. **After:** all cases pass. Those manifests now fail in `FetchManifest` with the errors above, before any file is written. - Accepted cases: `SKILL.md`, `references/auth.md`, names with digits, exactly 64 characters, and names that only contain a device name, such as `console` and `nul-tools`. - The published manifests `v0.2.32`, `v0.2.49` and `main` (35–38 skills, 366–420 files) contain no entry the check rejects. - `go build ./...`, `go test ./libs/aitools/... ./cmd/aitools/...`, the `aitools` acceptance tests and `./task lint-q` pass. ## Merge Danger **Door:** two-way. Rollback: revert the PR. **Blast Radius:** narrow. Only `aitools install`/`update` are affected, and only for manifests that contain a rejected name or file. Those commands now fail instead of writing. No published manifest is affected. A future manifest with a skill name outside the Agent Skills format would be rejected as a whole. Co-authored-by: Isaac --- .../cli/aitools-manifest-path-validation.md | 1 + libs/aitools/installer/source.go | 38 ++++++++ libs/aitools/installer/source_test.go | 88 +++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 .nextchanges/cli/aitools-manifest-path-validation.md diff --git a/.nextchanges/cli/aitools-manifest-path-validation.md b/.nextchanges/cli/aitools-manifest-path-validation.md new file mode 100644 index 00000000000..351f23496af --- /dev/null +++ b/.nextchanges/cli/aitools-manifest-path-validation.md @@ -0,0 +1 @@ +* `databricks aitools install` and `databricks aitools update` now reject a skills manifest whose skill names or file paths would write outside the skills directory, and fail before writing any files. ([#7722](https://github.com/databricks/cli/pull/7722)) diff --git a/libs/aitools/installer/source.go b/libs/aitools/installer/source.go index 27721de3f3b..ebea8cd2ee5 100644 --- a/libs/aitools/installer/source.go +++ b/libs/aitools/installer/source.go @@ -6,6 +6,8 @@ import ( "fmt" "io" "net/http" + "path/filepath" + "regexp" "time" "github.com/databricks/cli/libs/clicompat" @@ -63,10 +65,46 @@ func (s *GitHubManifestSource) FetchManifest(ctx context.Context, ref string) (* return nil, &FetchError{Stage: FetchStageManifest, StatusCode: resp.StatusCode, Err: fmt.Errorf("failed to parse manifest: %w", err)} } + if err := validateManifestPaths(&manifest); err != nil { + return nil, &FetchError{Stage: FetchStageManifest, StatusCode: resp.StatusCode, Err: fmt.Errorf("invalid skills manifest: %w", err)} + } normalizeManifest(&manifest) return &manifest, nil } +// skillNamePattern is the Agent Skills name format +// (https://agentskills.io/specification#name-field): lowercase letters, digits +// and single hyphens. Every name is a directory under the skills dir, next to +// the installer's state file, so the format also keeps names from aliasing each +// other or that file on case-insensitive filesystems or through Windows +// trailing dots and spaces. +var skillNamePattern = regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`) + +// windowsDeviceNamePattern matches the names Windows resolves to a device in +// any directory. They are rejected on every OS so a manifest is valid or +// invalid the same way everywhere. +var windowsDeviceNamePattern = regexp.MustCompile(`^(con|prn|aux|nul|com[1-9]|lpt[1-9])$`) + +// validateManifestPaths rejects skill names and files that would be written +// outside the skill's own directory. The manifest comes from the network, so it +// must not be able to name ../../.bashrc. +func validateManifestPaths(m *Manifest) error { + for name, meta := range m.Skills { + if len(name) > 64 || !skillNamePattern.MatchString(name) { + return fmt.Errorf("skill name %q must be at most 64 lowercase letters, digits and single hyphens", name) + } + if windowsDeviceNamePattern.MatchString(name) { + return fmt.Errorf("skill name %q is a reserved Windows device name", name) + } + for _, file := range meta.Files { + if !filepath.IsLocal(file) { + return fmt.Errorf("skill %q lists file %q outside its directory", name, file) + } + } + } + return nil +} + // normalizeManifest stamps SourceName and defaults RepoDir for older manifests. func normalizeManifest(m *Manifest) { if m.Skills == nil { diff --git a/libs/aitools/installer/source_test.go b/libs/aitools/installer/source_test.go index a0d38c64459..4aba06460ea 100644 --- a/libs/aitools/installer/source_test.go +++ b/libs/aitools/installer/source_test.go @@ -2,11 +2,16 @@ package installer import ( "errors" + "fmt" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/cmdio" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -96,3 +101,86 @@ func TestFetchManifestSizeCap(t *testing.T) { }) } } + +// TestFetchManifestRejectsPathsOutsideSkillDir covers manifest entries that +// would make an install write outside the skill's own directory: every skill +// name and file in the fetched manifest becomes a local path. +func TestFetchManifestRejectsPathsOutsideSkillDir(t *testing.T) { + tests := []struct { + name string + skill string + file string + wantErr string + }{ + {name: "regular file", skill: "databricks-core", file: "SKILL.md"}, + {name: "nested file", skill: "databricks-core", file: "references/auth.md"}, + {name: "file escaping the skill dir", skill: "databricks-core", file: "../../../.bashrc", wantErr: `skill "databricks-core" lists file "../../../.bashrc" outside its directory`}, + {name: "absolute file", skill: "databricks-core", file: "/etc/passwd", wantErr: `skill "databricks-core" lists file "/etc/passwd" outside its directory`}, + {name: "file escaping after a subdirectory", skill: "databricks-core", file: "references/../../x", wantErr: `skill "databricks-core" lists file "references/../../x" outside its directory`}, + {name: "name with digits", skill: "ai2-functions", file: "SKILL.md"}, + {name: "64-character name", skill: strings.Repeat("a", 64), file: "SKILL.md"}, + {name: "skill name escaping the skills dir", skill: "../.ssh", file: "SKILL.md", wantErr: `skill name "../.ssh" must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "nested skill name", skill: "a/b", file: "SKILL.md", wantErr: `skill name "a/b" must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "dot skill name", skill: ".", file: "SKILL.md", wantErr: `skill name "." must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "installer state file name", skill: ".state.json", file: "SKILL.md", wantErr: `skill name ".state.json" must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "uppercase name", skill: "Databricks-Core", file: "SKILL.md", wantErr: `skill name "Databricks-Core" must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "trailing dot", skill: "databricks-core.", file: "SKILL.md", wantErr: `skill name "databricks-core." must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "trailing space", skill: "databricks-core ", file: "SKILL.md", wantErr: `skill name "databricks-core " must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "underscore", skill: "databricks_core", file: "SKILL.md", wantErr: `skill name "databricks_core" must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "leading hyphen", skill: "-databricks", file: "SKILL.md", wantErr: `skill name "-databricks" must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "double hyphen", skill: "databricks--core", file: "SKILL.md", wantErr: `skill name "databricks--core" must be at most 64 lowercase letters, digits and single hyphens`}, + {name: "Windows device name nul", skill: "nul", file: "SKILL.md", wantErr: `skill name "nul" is a reserved Windows device name`}, + {name: "Windows device name con", skill: "con", file: "SKILL.md", wantErr: `skill name "con" is a reserved Windows device name`}, + {name: "Windows device name com1", skill: "com1", file: "SKILL.md", wantErr: `skill name "com1" is a reserved Windows device name`}, + {name: "Windows device name lpt9", skill: "lpt9", file: "SKILL.md", wantErr: `skill name "lpt9" is a reserved Windows device name`}, + {name: "device name prefix", skill: "console", file: "SKILL.md"}, + {name: "device name with a suffix", skill: "nul-tools", file: "SKILL.md"}, + {name: "65-character name", skill: strings.Repeat("a", 65), file: "SKILL.md", wantErr: fmt.Sprintf(`skill name %q must be at most 64 lowercase letters, digits and single hyphens`, strings.Repeat("a", 65))}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + body := fmt.Sprintf(`{"version":"1","skills":{%q:{"version":"0.1.0","files":[%q]}}}`, tc.skill, tc.file) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) + + manifest, err := (&GitHubManifestSource{}).FetchManifest(t.Context(), "v1") + if tc.wantErr == "" { + require.NoError(t, err) + assert.Equal(t, []string{tc.file}, manifest.Skills[tc.skill].Files) + return + } + assert.EqualError(t, err, "invalid skills manifest: "+tc.wantErr) + _, ok := errors.AsType[*FetchError](err) + assert.True(t, ok) + }) + } +} + +func TestInstallRejectsStateFileSkillNameWithoutTouchingState(t *testing.T) { + tmp := setupTestHome(t) + ctx := cmdio.MockDiscard(t.Context()) + setupFetchMock(t) + t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef) + agent := testAgent(tmp) + require.NoError(t, InstallSkillsForAgents(ctx, &mockManifestSource{manifest: testManifest()}, []*agents.Agent{agent}, InstallOptions{})) + + statePath := filepath.Join(tmp, ".databricks", "aitools", "skills", stateFileName) + before, err := os.ReadFile(statePath) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"version":"1","skills":{".state.json":{"version":"0.1.0","files":["SKILL.md"]}}}`)) + })) + t.Cleanup(server.Close) + t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) + + err = InstallSkillsForAgents(ctx, &GitHubManifestSource{}, []*agents.Agent{agent}, InstallOptions{}) + require.ErrorContains(t, err, `skill name ".state.json"`) + + after, err := os.ReadFile(statePath) + require.NoError(t, err) + assert.Equal(t, string(before), string(after)) +} From 51002ef06a9e583afb37353b57a6dbabeb6b6da7 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:34:22 +0000 Subject: [PATCH 2/4] aitools: fail only on manifest escapes, skip unportable skills, classify as MANIFEST_INVALID Addresses review on #7722: - An escaping name or file now wraps ErrUnsafeManifestPath, classified as MANIFEST_INVALID with its own recovery step instead of SKILLS_FETCH_FAILED. - Only real escapes (absolute, drive, too many "..", separators in a name, ".", "..", "", the state file name) fail the manifest. A name outside the Agent Skills format, a device name, or a non-clean file path drops only that skill with a warning, like a min_cli_version mismatch. - The check moved to FetchSkillsManifestWithFallback, so every ManifestSource goes through it. - File rules are OS-independent (both slashes, colons, device-name elements). - Lazy regexp, sorted iteration, maxSkillNameLength constant, wider changelog. Co-authored-by: Isaac --- .../cli/aitools-manifest-path-validation.md | 2 +- cmd/aitools/categorize.go | 5 + cmd/aitools/categorize_test.go | 15 ++ cmd/aitools/recovery.go | 3 + cmd/aitools/recovery_test.go | 5 + libs/aitools/installer/errors.go | 5 + libs/aitools/installer/installer.go | 8 +- libs/aitools/installer/source.go | 121 ++++++++++++---- libs/aitools/installer/source_test.go | 130 ++++++++++++------ 9 files changed, 225 insertions(+), 69 deletions(-) diff --git a/.nextchanges/cli/aitools-manifest-path-validation.md b/.nextchanges/cli/aitools-manifest-path-validation.md index 351f23496af..e5c0289bc23 100644 --- a/.nextchanges/cli/aitools-manifest-path-validation.md +++ b/.nextchanges/cli/aitools-manifest-path-validation.md @@ -1 +1 @@ -* `databricks aitools install` and `databricks aitools update` now reject a skills manifest whose skill names or file paths would write outside the skills directory, and fail before writing any files. ([#7722](https://github.com/databricks/cli/pull/7722)) +* `databricks aitools install`, `update` and `list` now refuse a skills manifest that names a skill or file outside the skills directory, before writing any files, and skip with a warning a skill whose name or file paths are not portable. ([#7722](https://github.com/databricks/cli/pull/7722)) diff --git a/cmd/aitools/categorize.go b/cmd/aitools/categorize.go index bae47fe0e3f..27572da2176 100644 --- a/cmd/aitools/categorize.go +++ b/cmd/aitools/categorize.go @@ -72,6 +72,11 @@ func classifyInstallError(err error) protos.AitoolsErrorCategory { if errors.Is(err, installer.ErrUnknownAgent) { return protos.AitoolsErrorCategoryUnknownAgent } + // The manifest arrived and parsed, but names a path outside the skills + // directory: content, not transport. + if errors.Is(err, installer.ErrUnsafeManifestPath) { + return protos.AitoolsErrorCategoryManifestInvalid + } // Skills on disk without the install state that update and uninstall need. if errors.Is(err, installer.ErrLegacyInstall) { return protos.AitoolsErrorCategoryStateInvalid diff --git a/cmd/aitools/categorize_test.go b/cmd/aitools/categorize_test.go index ce58f809a0b..fbfa99603e6 100644 --- a/cmd/aitools/categorize_test.go +++ b/cmd/aitools/categorize_test.go @@ -325,3 +325,18 @@ func TestClassifyRealManifestFetchErrors(t *testing.T) { }) } } + +// TestClassifyUnsafeManifestPath covers a manifest that downloads and parses +// but names a file outside the skills directory: an invalid manifest, not a +// failed download. +func TestClassifyUnsafeManifestPath(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"version":"1","skills":{"databricks-core":{"version":"0.1.0","files":["../../.bashrc"]}}}`)) + })) + t.Cleanup(server.Close) + t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) + + _, _, err := installer.FetchSkillsManifestWithFallback(t.Context(), &installer.GitHubManifestSource{}, "v1", false) + require.ErrorIs(t, err, installer.ErrUnsafeManifestPath) + assert.Equal(t, protos.AitoolsErrorCategoryManifestInvalid, classifyInstallError(err)) +} diff --git a/cmd/aitools/recovery.go b/cmd/aitools/recovery.go index ebb02260ca3..1df231d08f0 100644 --- a/cmd/aitools/recovery.go +++ b/cmd/aitools/recovery.go @@ -238,6 +238,9 @@ func installRecovery(ctx context.Context, inv installInvocation, err error) stri return "The skills release this CLI selected is missing. Set DATABRICKS_SKILLS_REF to a release tag from " + skillsReleasesURL + " and re-run " + rerun + ", and report it at " + cliIssuesURL + "." case protos.AitoolsErrorCategoryManifestInvalid: host := skillsHost(ctx) + if errors.Is(err, installer.ErrUnsafeManifestPath) { + return "No files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from " + skillsReleasesURL + " and re-run " + rerun + ", and report the manifest from " + host + " at " + cliIssuesURL + "." + } return host + " answered with something that is not a skills manifest (a proxy or captive portal page, or a broken mirror). Check what this machine gets from " + host + ", then re-run " + rerun + "." case protos.AitoolsErrorCategorySkillsFetchFailed: return downloadRecovery(ctx, inv, ferr) diff --git a/cmd/aitools/recovery_test.go b/cmd/aitools/recovery_test.go index 5c7852d865a..0bd1c6f56c7 100644 --- a/cmd/aitools/recovery_test.go +++ b/cmd/aitools/recovery_test.go @@ -70,6 +70,11 @@ func TestInstallRecovery(t *testing.T) { err: &installer.FetchError{Stage: installer.FetchStageManifest, StatusCode: http.StatusOK, Err: fmt.Errorf("failed to parse manifest: %w", &json.SyntaxError{})}, want: "raw.githubusercontent.com answered with something that is not a skills manifest (a proxy or captive portal page, or a broken mirror). Check what this machine gets from raw.githubusercontent.com, then re-run 'databricks aitools install'.", }, + { + name: "manifest names a path outside the skills directory", + err: fmt.Errorf("%w: skill %q lists file %q", installer.ErrUnsafeManifestPath, "databricks-core", "../../.bashrc"), + want: "No files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from " + skillsReleasesURL + " and re-run 'databricks aitools install', and report the manifest from raw.githubusercontent.com at " + cliIssuesURL + ".", + }, { name: "manifest fetch without a response", err: &installer.FetchError{Stage: installer.FetchStageManifest, Err: fmt.Errorf("failed to fetch manifest: %w", dnsError)}, diff --git a/libs/aitools/installer/errors.go b/libs/aitools/installer/errors.go index 25e41f94e74..b860d474531 100644 --- a/libs/aitools/installer/errors.go +++ b/libs/aitools/installer/errors.go @@ -76,6 +76,11 @@ const maxFetchBodySize = 1 << 20 // The command layer classifies it like a manifest that does not parse. var ErrManifestTooLarge = fmt.Errorf("skills manifest is larger than %d MiB", maxFetchBodySize>>20) +// ErrUnsafeManifestPath reports a skills manifest that names a skill or file +// outside the skills directory. The command layer classifies it like a +// manifest that does not parse. +var ErrUnsafeManifestPath = errors.New("skills manifest names a path outside the skills directory") + // Stages of fetching skills content, recorded on FetchError so a failed // release lookup can be told apart from a failed download. const ( diff --git a/libs/aitools/installer/installer.go b/libs/aitools/installer/installer.go index 6b2c797ac5d..519980f5c55 100644 --- a/libs/aitools/installer/installer.go +++ b/libs/aitools/installer/installer.go @@ -274,7 +274,13 @@ func FetchSkillsManifestWithFallback(ctx context.Context, src ManifestSource, re log.Warnf(ctx, "Could not resolve embedded skills version: %v", fbErr) } } - return manifest, ref, err + if err != nil { + return nil, ref, err + } + if err := checkManifestPaths(ctx, manifest); err != nil { + return nil, ref, err + } + return manifest, ref, nil } // installedSkillsRelease returns the newest recorded raw-skills release and diff --git a/libs/aitools/installer/source.go b/libs/aitools/installer/source.go index ebea8cd2ee5..b0c455dea80 100644 --- a/libs/aitools/installer/source.go +++ b/libs/aitools/installer/source.go @@ -5,9 +5,13 @@ import ( "encoding/json" "fmt" "io" + "maps" "net/http" - "path/filepath" + "path" "regexp" + "slices" + "strings" + "sync" "time" "github.com/databricks/cli/libs/clicompat" @@ -65,46 +69,111 @@ func (s *GitHubManifestSource) FetchManifest(ctx context.Context, ref string) (* return nil, &FetchError{Stage: FetchStageManifest, StatusCode: resp.StatusCode, Err: fmt.Errorf("failed to parse manifest: %w", err)} } - if err := validateManifestPaths(&manifest); err != nil { - return nil, &FetchError{Stage: FetchStageManifest, StatusCode: resp.StatusCode, Err: fmt.Errorf("invalid skills manifest: %w", err)} - } normalizeManifest(&manifest) return &manifest, nil } +// maxSkillNameLength is the Agent Skills limit on the length of a skill name. +const maxSkillNameLength = 64 + // skillNamePattern is the Agent Skills name format // (https://agentskills.io/specification#name-field): lowercase letters, digits // and single hyphens. Every name is a directory under the skills dir, next to // the installer's state file, so the format also keeps names from aliasing each -// other or that file on case-insensitive filesystems or through Windows -// trailing dots and spaces. -var skillNamePattern = regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`) - -// windowsDeviceNamePattern matches the names Windows resolves to a device in -// any directory. They are rejected on every OS so a manifest is valid or -// invalid the same way everywhere. -var windowsDeviceNamePattern = regexp.MustCompile(`^(con|prn|aux|nul|com[1-9]|lpt[1-9])$`) - -// validateManifestPaths rejects skill names and files that would be written -// outside the skill's own directory. The manifest comes from the network, so it -// must not be able to name ../../.bashrc. -func validateManifestPaths(m *Manifest) error { - for name, meta := range m.Skills { - if len(name) > 64 || !skillNamePattern.MatchString(name) { - return fmt.Errorf("skill name %q must be at most 64 lowercase letters, digits and single hyphens", name) - } - if windowsDeviceNamePattern.MatchString(name) { - return fmt.Errorf("skill name %q is a reserved Windows device name", name) +// other on case-insensitive filesystems or through Windows trailing dots and +// spaces. +var skillNamePattern = sync.OnceValue(func() *regexp.Regexp { + return regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`) +}) + +// checkManifestPaths makes every skill name and file in m safe to join onto a +// local directory. The manifest comes from the network, so it must not be able +// to name ../../.bashrc. A name or file that reaches outside its directory, or +// a skill named like the state file, fails the whole manifest: only a broken or +// tampered manifest lists one. A skill whose name or files are merely not +// portable is dropped with a warning, the way resolveSkills skips a skill this +// CLI is too old for, so one such entry does not break every install. Both +// rules are the same on every OS, so a manifest is treated the same everywhere. +func checkManifestPaths(ctx context.Context, m *Manifest) error { + names := slices.Sorted(maps.Keys(m.Skills)) + for _, name := range names { + if name == "" || name == "." || name == ".." || name == stateFileName || strings.ContainsAny(name, `/\`) { + return fmt.Errorf("%w: skill name %q", ErrUnsafeManifestPath, name) } - for _, file := range meta.Files { - if !filepath.IsLocal(file) { - return fmt.Errorf("skill %q lists file %q outside its directory", name, file) + for _, file := range m.Skills[name].Files { + if escapesDir(file) { + return fmt.Errorf("%w: skill %q lists file %q", ErrUnsafeManifestPath, name, file) } } } + for _, name := range names { + if reason := unportableSkill(name, m.Skills[name].Files); reason != "" { + log.Warnf(ctx, "Skipping skill %s: %s", name, reason) + delete(m.Skills, name) + } + } return nil } +// escapesDir reports whether file, joined onto a directory, names a path +// outside it on some OS: an absolute path, a Windows drive, or more ".." +// elements than the elements before them, with either slash as the separator. +func escapesDir(file string) bool { + if strings.HasPrefix(file, "/") || strings.HasPrefix(file, `\`) || (len(file) >= 2 && file[1] == ':') { + return true + } + depth := 0 + for _, elem := range strings.FieldsFunc(file, func(r rune) bool { return r == '/' || r == '\\' }) { + switch elem { + case ".": + case "..": + depth-- + if depth < 0 { + return true + } + default: + depth++ + } + } + return false +} + +// unportableSkill returns why a skill that stays inside its directory still +// cannot be installed the same way on every OS, or "" if it can. Files must be +// clean slash-separated paths, so the paths recorded in the install state match +// the ones update walks on disk. +func unportableSkill(name string, files []string) string { + if len(name) > maxSkillNameLength || !skillNamePattern().MatchString(name) { + return fmt.Sprintf("the name is not at most %d lowercase letters, digits and single hyphens", maxSkillNameLength) + } + if isWindowsDeviceName(name) { + return "the name is a reserved Windows device name" + } + for _, file := range files { + if file == "." || path.Clean(file) != file || strings.ContainsAny(file, `\:`) { + return fmt.Sprintf("file %q is not a clean slash-separated path", file) + } + for elem := range strings.SplitSeq(file, "/") { + stem, _, _ := strings.Cut(elem, ".") + if isWindowsDeviceName(stem) { + return fmt.Sprintf("file %q names a reserved Windows device", file) + } + } + } + return "" +} + +// isWindowsDeviceName reports whether s, in any case, is a name Windows +// resolves to a device in every directory. +func isWindowsDeviceName(s string) bool { + s = strings.ToLower(s) + switch s { + case "con", "prn", "aux", "nul": + return true + } + return len(s) == 4 && (strings.HasPrefix(s, "com") || strings.HasPrefix(s, "lpt")) && s[3] >= '1' && s[3] <= '9' +} + // normalizeManifest stamps SourceName and defaults RepoDir for older manifests. func normalizeManifest(m *Manifest) { if m.Skills == nil { diff --git a/libs/aitools/installer/source_test.go b/libs/aitools/installer/source_test.go index 4aba06460ea..dac0c65c7a9 100644 --- a/libs/aitools/installer/source_test.go +++ b/libs/aitools/installer/source_test.go @@ -2,11 +2,12 @@ package installer import ( "errors" - "fmt" + "maps" "net/http" "net/http/httptest" "os" "path/filepath" + "slices" "strings" "testing" @@ -102,63 +103,110 @@ func TestFetchManifestSizeCap(t *testing.T) { } } -// TestFetchManifestRejectsPathsOutsideSkillDir covers manifest entries that -// would make an install write outside the skill's own directory: every skill -// name and file in the fetched manifest becomes a local path. -func TestFetchManifestRejectsPathsOutsideSkillDir(t *testing.T) { +// TestCheckManifestPaths covers every skill name and file in a fetched +// manifest becoming a local path: an escape fails the manifest, an entry that +// is not portable drops only its skill. +func TestCheckManifestPaths(t *testing.T) { tests := []struct { - name string - skill string - file string + name string + skill string + file string + // wantErr is the error after the ErrUnsafeManifestPath prefix. wantErr string + // wantSkipped is true when the skill is dropped from the manifest. + wantSkipped bool }{ {name: "regular file", skill: "databricks-core", file: "SKILL.md"}, {name: "nested file", skill: "databricks-core", file: "references/auth.md"}, - {name: "file escaping the skill dir", skill: "databricks-core", file: "../../../.bashrc", wantErr: `skill "databricks-core" lists file "../../../.bashrc" outside its directory`}, - {name: "absolute file", skill: "databricks-core", file: "/etc/passwd", wantErr: `skill "databricks-core" lists file "/etc/passwd" outside its directory`}, - {name: "file escaping after a subdirectory", skill: "databricks-core", file: "references/../../x", wantErr: `skill "databricks-core" lists file "references/../../x" outside its directory`}, {name: "name with digits", skill: "ai2-functions", file: "SKILL.md"}, {name: "64-character name", skill: strings.Repeat("a", 64), file: "SKILL.md"}, - {name: "skill name escaping the skills dir", skill: "../.ssh", file: "SKILL.md", wantErr: `skill name "../.ssh" must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "nested skill name", skill: "a/b", file: "SKILL.md", wantErr: `skill name "a/b" must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "dot skill name", skill: ".", file: "SKILL.md", wantErr: `skill name "." must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "installer state file name", skill: ".state.json", file: "SKILL.md", wantErr: `skill name ".state.json" must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "uppercase name", skill: "Databricks-Core", file: "SKILL.md", wantErr: `skill name "Databricks-Core" must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "trailing dot", skill: "databricks-core.", file: "SKILL.md", wantErr: `skill name "databricks-core." must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "trailing space", skill: "databricks-core ", file: "SKILL.md", wantErr: `skill name "databricks-core " must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "underscore", skill: "databricks_core", file: "SKILL.md", wantErr: `skill name "databricks_core" must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "leading hyphen", skill: "-databricks", file: "SKILL.md", wantErr: `skill name "-databricks" must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "double hyphen", skill: "databricks--core", file: "SKILL.md", wantErr: `skill name "databricks--core" must be at most 64 lowercase letters, digits and single hyphens`}, - {name: "Windows device name nul", skill: "nul", file: "SKILL.md", wantErr: `skill name "nul" is a reserved Windows device name`}, - {name: "Windows device name con", skill: "con", file: "SKILL.md", wantErr: `skill name "con" is a reserved Windows device name`}, - {name: "Windows device name com1", skill: "com1", file: "SKILL.md", wantErr: `skill name "com1" is a reserved Windows device name`}, - {name: "Windows device name lpt9", skill: "lpt9", file: "SKILL.md", wantErr: `skill name "lpt9" is a reserved Windows device name`}, {name: "device name prefix", skill: "console", file: "SKILL.md"}, {name: "device name with a suffix", skill: "nul-tools", file: "SKILL.md"}, - {name: "65-character name", skill: strings.Repeat("a", 65), file: "SKILL.md", wantErr: fmt.Sprintf(`skill name %q must be at most 64 lowercase letters, digits and single hyphens`, strings.Repeat("a", 65))}, + {name: "file with a device name prefix", skill: "databricks-core", file: "references/console.md"}, + + {name: "file escaping the skill dir", skill: "databricks-core", file: "../../../.bashrc", wantErr: `skill "databricks-core" lists file "../../../.bashrc"`}, + {name: "absolute file", skill: "databricks-core", file: "/etc/passwd", wantErr: `skill "databricks-core" lists file "/etc/passwd"`}, + {name: "file escaping after a subdirectory", skill: "databricks-core", file: "references/../../x", wantErr: `skill "databricks-core" lists file "references/../../x"`}, + {name: "file escaping with backslashes", skill: "databricks-core", file: `a\..\..\x`, wantErr: `skill "databricks-core" lists file "a\\..\\..\\x"`}, + {name: "file rooted with a backslash", skill: "databricks-core", file: `\x`, wantErr: `skill "databricks-core" lists file "\\x"`}, + {name: "file on a Windows drive", skill: "databricks-core", file: "C:x", wantErr: `skill "databricks-core" lists file "C:x"`}, + {name: "skill name escaping the skills dir", skill: "../.ssh", file: "SKILL.md", wantErr: `skill name "../.ssh"`}, + {name: "nested skill name", skill: "a/b", file: "SKILL.md", wantErr: `skill name "a/b"`}, + {name: "skill name with a backslash", skill: `a\b`, file: "SKILL.md", wantErr: `skill name "a\\b"`}, + {name: "dot skill name", skill: ".", file: "SKILL.md", wantErr: `skill name "."`}, + {name: "empty skill name", skill: "", file: "SKILL.md", wantErr: `skill name ""`}, + {name: "installer state file name", skill: ".state.json", file: "SKILL.md", wantErr: `skill name ".state.json"`}, + + {name: "uppercase name", skill: "Databricks-Core", file: "SKILL.md", wantSkipped: true}, + {name: "trailing dot", skill: "databricks-core.", file: "SKILL.md", wantSkipped: true}, + {name: "trailing space", skill: "databricks-core ", file: "SKILL.md", wantSkipped: true}, + {name: "underscore", skill: "databricks_core", file: "SKILL.md", wantSkipped: true}, + {name: "leading hyphen", skill: "-databricks", file: "SKILL.md", wantSkipped: true}, + {name: "double hyphen", skill: "databricks--core", file: "SKILL.md", wantSkipped: true}, + {name: "65-character name", skill: strings.Repeat("a", 65), file: "SKILL.md", wantSkipped: true}, + {name: "Windows device name nul", skill: "nul", file: "SKILL.md", wantSkipped: true}, + {name: "Windows device name com1", skill: "com1", file: "SKILL.md", wantSkipped: true}, + {name: "Windows device name lpt9", skill: "lpt9", file: "SKILL.md", wantSkipped: true}, + {name: "file is the skill dir", skill: "databricks-core", file: ".", wantSkipped: true}, + {name: "file resolving to the skill dir", skill: "databricks-core", file: "references/..", wantSkipped: true}, + {name: "file with a dot element", skill: "databricks-core", file: "./SKILL.md", wantSkipped: true}, + {name: "file with a .. element that stays inside", skill: "databricks-core", file: "references/../SKILL.md", wantSkipped: true}, + {name: "file with a double slash", skill: "databricks-core", file: "references//auth.md", wantSkipped: true}, + {name: "empty file", skill: "databricks-core", file: "", wantSkipped: true}, + {name: "file with a backslash", skill: "databricks-core", file: `references\auth.md`, wantSkipped: true}, + {name: "file with a colon", skill: "databricks-core", file: "notes:v2.md", wantSkipped: true}, + {name: "file named like a device", skill: "databricks-core", file: "references/aux.md", wantSkipped: true}, + {name: "directory named like a device", skill: "databricks-core", file: "CON/SKILL.md", wantSkipped: true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - body := fmt.Sprintf(`{"version":"1","skills":{%q:{"version":"0.1.0","files":[%q]}}}`, tc.skill, tc.file) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(body)) - })) - t.Cleanup(server.Close) - t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) - - manifest, err := (&GitHubManifestSource{}).FetchManifest(t.Context(), "v1") - if tc.wantErr == "" { - require.NoError(t, err) - assert.Equal(t, []string{tc.file}, manifest.Skills[tc.skill].Files) + m := &Manifest{Skills: map[string]SkillMeta{ + tc.skill: {Files: []string{tc.file}}, + "other-skill": {Files: []string{"SKILL.md"}}, + }} + err := checkManifestPaths(t.Context(), m) + if tc.wantErr != "" { + require.ErrorIs(t, err, ErrUnsafeManifestPath) + assert.EqualError(t, err, ErrUnsafeManifestPath.Error()+": "+tc.wantErr) return } - assert.EqualError(t, err, "invalid skills manifest: "+tc.wantErr) - _, ok := errors.AsType[*FetchError](err) - assert.True(t, ok) + require.NoError(t, err) + _, kept := m.Skills[tc.skill] + assert.Equal(t, !tc.wantSkipped, kept) + assert.Contains(t, m.Skills, "other-skill") }) } } +func TestCheckManifestPathsReportsTheFirstSkillInOrder(t *testing.T) { + m := &Manifest{Skills: map[string]SkillMeta{ + "b-skill": {Files: []string{"../b"}}, + "a-skill": {Files: []string{"../a"}}, + "../c": {Files: []string{"SKILL.md"}}, + }} + for range 10 { + assert.EqualError(t, checkManifestPaths(t.Context(), m), ErrUnsafeManifestPath.Error()+`: skill name "../c"`) + } +} + +// TestFetchSkillsManifestChecksEverySource checks the manifest at the point +// every command fetches it through, whatever the source. +func TestFetchSkillsManifestChecksEverySource(t *testing.T) { + src := &mockManifestSource{manifest: &Manifest{Skills: map[string]SkillMeta{ + "databricks-core": {Files: []string{"../../.bashrc"}}, + }}} + _, _, err := FetchSkillsManifestWithFallback(t.Context(), src, "v1", false) + require.ErrorIs(t, err, ErrUnsafeManifestPath) + + src = &mockManifestSource{manifest: &Manifest{Skills: map[string]SkillMeta{ + "databricks-core": {Files: []string{"SKILL.md"}}, + "Databricks_Sql": {Files: []string{"SKILL.md"}}, + }}} + manifest, _, err := FetchSkillsManifestWithFallback(t.Context(), src, "v1", false) + require.NoError(t, err) + assert.Equal(t, []string{"databricks-core"}, slices.Collect(maps.Keys(manifest.Skills))) +} + func TestInstallRejectsStateFileSkillNameWithoutTouchingState(t *testing.T) { tmp := setupTestHome(t) ctx := cmdio.MockDiscard(t.Context()) @@ -178,7 +226,7 @@ func TestInstallRejectsStateFileSkillNameWithoutTouchingState(t *testing.T) { t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) err = InstallSkillsForAgents(ctx, &GitHubManifestSource{}, []*agents.Agent{agent}, InstallOptions{}) - require.ErrorContains(t, err, `skill name ".state.json"`) + require.ErrorIs(t, err, ErrUnsafeManifestPath) after, err := os.ReadFile(statePath) require.NoError(t, err) From 02fa672bf60b6aae02d21ff8bfd25c56812c5f89 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:53:02 +0000 Subject: [PATCH 3/4] aitools: skip unportable skills in resolveSkills and update, not in the fetched manifest Addresses the second review on #7722: - FetchSkillsManifestWithFallback keeps only the escape check. unportableSkill now runs where min_cli_version is applied: resolveSkills skips the skill (a SkillError with ReasonUnportableSkill for --skills, MANIFEST_INVALID), and UpdateSkills reports it as Skipped, keeping the installed copy instead of pruning it. - File path elements are limited to [A-Za-z0-9._-] with no trailing dot, and files that differ only in case skip the skill. - Warnings quote the skill name; the recovery step and changelog say "No skill files were written"; the FetchSkillsManifestWithFallback doc comment names the check. Co-authored-by: Isaac --- .../cli/aitools-manifest-path-validation.md | 2 +- cmd/aitools/categorize.go | 3 + cmd/aitools/categorize_test.go | 5 ++ cmd/aitools/recovery.go | 6 +- cmd/aitools/recovery_test.go | 7 +- libs/aitools/installer/errors.go | 3 + libs/aitools/installer/installer.go | 15 ++++- libs/aitools/installer/source.go | 51 +++++++++------ libs/aitools/installer/source_test.go | 64 ++++++++++++------- libs/aitools/installer/update.go | 8 +++ libs/aitools/installer/update_test.go | 33 ++++++++++ 11 files changed, 147 insertions(+), 50 deletions(-) diff --git a/.nextchanges/cli/aitools-manifest-path-validation.md b/.nextchanges/cli/aitools-manifest-path-validation.md index e5c0289bc23..45b6dde1f7d 100644 --- a/.nextchanges/cli/aitools-manifest-path-validation.md +++ b/.nextchanges/cli/aitools-manifest-path-validation.md @@ -1 +1 @@ -* `databricks aitools install`, `update` and `list` now refuse a skills manifest that names a skill or file outside the skills directory, before writing any files, and skip with a warning a skill whose name or file paths are not portable. ([#7722](https://github.com/databricks/cli/pull/7722)) +* `databricks aitools install`, `update` and `list` now refuse a skills manifest that names a skill or file outside the skills directory, before writing any skill files. A skill whose name or file paths are not portable is skipped with a warning, and `update` keeps its installed copy. ([#7722](https://github.com/databricks/cli/pull/7722)) diff --git a/cmd/aitools/categorize.go b/cmd/aitools/categorize.go index 27572da2176..aab075edb07 100644 --- a/cmd/aitools/categorize.go +++ b/cmd/aitools/categorize.go @@ -142,6 +142,9 @@ func skillErrorCategory(e *installer.SkillError) protos.AitoolsErrorCategory { return protos.AitoolsErrorCategoryVersionIncompatible case installer.ReasonExperimentalSkill: return protos.AitoolsErrorCategoryExperimentalSkill + // The manifest lists the skill in a form it cannot be installed in. + case installer.ReasonUnportableSkill: + return protos.AitoolsErrorCategoryManifestInvalid default: return protos.AitoolsErrorCategoryUncategorized } diff --git a/cmd/aitools/categorize_test.go b/cmd/aitools/categorize_test.go index fbfa99603e6..cc79f24c6bd 100644 --- a/cmd/aitools/categorize_test.go +++ b/cmd/aitools/categorize_test.go @@ -340,3 +340,8 @@ func TestClassifyUnsafeManifestPath(t *testing.T) { require.ErrorIs(t, err, installer.ErrUnsafeManifestPath) assert.Equal(t, protos.AitoolsErrorCategoryManifestInvalid, classifyInstallError(err)) } + +func TestClassifyUnportableSkill(t *testing.T) { + err := &installer.SkillError{Skill: "Databricks_Core", Reason: installer.ReasonUnportableSkill} + assert.Equal(t, protos.AitoolsErrorCategoryManifestInvalid, classifyInstallError(err)) +} diff --git a/cmd/aitools/recovery.go b/cmd/aitools/recovery.go index 1df231d08f0..c7998908e7f 100644 --- a/cmd/aitools/recovery.go +++ b/cmd/aitools/recovery.go @@ -238,8 +238,10 @@ func installRecovery(ctx context.Context, inv installInvocation, err error) stri return "The skills release this CLI selected is missing. Set DATABRICKS_SKILLS_REF to a release tag from " + skillsReleasesURL + " and re-run " + rerun + ", and report it at " + cliIssuesURL + "." case protos.AitoolsErrorCategoryManifestInvalid: host := skillsHost(ctx) - if errors.Is(err, installer.ErrUnsafeManifestPath) { - return "No files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from " + skillsReleasesURL + " and re-run " + rerun + ", and report the manifest from " + host + " at " + cliIssuesURL + "." + // The manifest arrived and decoded, but lists a path outside the + // skills directory or a skill in a form it cannot be installed in. + if _, ok := errors.AsType[*installer.SkillError](err); ok || errors.Is(err, installer.ErrUnsafeManifestPath) { + return "No skill files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from " + skillsReleasesURL + " and re-run " + rerun + ", and report the manifest from " + host + " at " + cliIssuesURL + "." } return host + " answered with something that is not a skills manifest (a proxy or captive portal page, or a broken mirror). Check what this machine gets from " + host + ", then re-run " + rerun + "." case protos.AitoolsErrorCategorySkillsFetchFailed: diff --git a/cmd/aitools/recovery_test.go b/cmd/aitools/recovery_test.go index 0bd1c6f56c7..9bc9c95f1d9 100644 --- a/cmd/aitools/recovery_test.go +++ b/cmd/aitools/recovery_test.go @@ -73,7 +73,12 @@ func TestInstallRecovery(t *testing.T) { { name: "manifest names a path outside the skills directory", err: fmt.Errorf("%w: skill %q lists file %q", installer.ErrUnsafeManifestPath, "databricks-core", "../../.bashrc"), - want: "No files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from " + skillsReleasesURL + " and re-run 'databricks aitools install', and report the manifest from raw.githubusercontent.com at " + cliIssuesURL + ".", + want: "No skill files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from " + skillsReleasesURL + " and re-run 'databricks aitools install', and report the manifest from raw.githubusercontent.com at " + cliIssuesURL + ".", + }, + { + name: "named skill is not portable", + err: &installer.SkillError{Skill: "Databricks_Core", Reason: installer.ReasonUnportableSkill, Detail: "cannot be installed"}, + want: "No skill files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from " + skillsReleasesURL + " and re-run 'databricks aitools install', and report the manifest from raw.githubusercontent.com at " + cliIssuesURL + ".", }, { name: "manifest fetch without a response", diff --git a/libs/aitools/installer/errors.go b/libs/aitools/installer/errors.go index b860d474531..0f4c9a5b539 100644 --- a/libs/aitools/installer/errors.go +++ b/libs/aitools/installer/errors.go @@ -57,6 +57,9 @@ const ( ReasonVersionIncompatible = "version-incompatible" // ReasonExperimentalSkill: the named skill is experimental and --experimental was not set. ReasonExperimentalSkill = "experimental-skill" + // ReasonUnportableSkill: the manifest lists the skill with a name or file + // path that is not portable, so it is never installed. + ReasonUnportableSkill = "unportable-skill" ) func (e *SkillError) Error() string { diff --git a/libs/aitools/installer/installer.go b/libs/aitools/installer/installer.go index 519980f5c55..f6f4156defa 100644 --- a/libs/aitools/installer/installer.go +++ b/libs/aitools/installer/installer.go @@ -259,8 +259,9 @@ func fetchSkillFile(ctx context.Context, ref, repoDir, skillName, filePath strin // FetchSkillsManifestWithFallback fetches the skills manifest at the given ref. // If the ref points to a non-existent tag (not-found error), it falls back to -// the embedded manifest's skills version. Returns the manifest, the (possibly -// updated) ref, and any error. +// the embedded manifest's skills version. A manifest that names a skill or file +// outside its directory fails with ErrUnsafeManifestPath, whatever the source. +// Returns the manifest, the (possibly updated) ref, and any error. func FetchSkillsManifestWithFallback(ctx context.Context, src ManifestSource, ref string, allowFallback bool) (*Manifest, string, error) { tag := strings.TrimPrefix(ref, "v") manifest, err := src.FetchManifest(ctx, ref) @@ -277,7 +278,7 @@ func FetchSkillsManifestWithFallback(ctx context.Context, src ManifestSource, re if err != nil { return nil, ref, err } - if err := checkManifestPaths(ctx, manifest); err != nil { + if err := checkManifestPaths(manifest); err != nil { return nil, ref, err } return manifest, ref, nil @@ -555,6 +556,14 @@ func resolveSkills(ctx context.Context, skills map[string]SkillMeta, opts Instal result := make(map[string]SkillMeta, len(candidates)) for name, meta := range candidates { + if reason := unportableSkill(name, meta.Files); reason != "" { + if isSpecific { + return nil, &SkillError{Skill: name, Reason: ReasonUnportableSkill, Detail: "cannot be installed: " + reason} + } + log.Warnf(ctx, "Skipping skill %q: %s", name, reason) + continue + } + if meta.IsExperimental() && !opts.IncludeExperimental { if isSpecific { return nil, &SkillError{Skill: name, Reason: ReasonExperimentalSkill, Detail: "is experimental; use --experimental to install"} diff --git a/libs/aitools/installer/source.go b/libs/aitools/installer/source.go index b0c455dea80..1a408e81369 100644 --- a/libs/aitools/installer/source.go +++ b/libs/aitools/installer/source.go @@ -86,17 +86,14 @@ var skillNamePattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`) }) -// checkManifestPaths makes every skill name and file in m safe to join onto a -// local directory. The manifest comes from the network, so it must not be able -// to name ../../.bashrc. A name or file that reaches outside its directory, or -// a skill named like the state file, fails the whole manifest: only a broken or -// tampered manifest lists one. A skill whose name or files are merely not -// portable is dropped with a warning, the way resolveSkills skips a skill this -// CLI is too old for, so one such entry does not break every install. Both -// rules are the same on every OS, so a manifest is treated the same everywhere. -func checkManifestPaths(ctx context.Context, m *Manifest) error { - names := slices.Sorted(maps.Keys(m.Skills)) - for _, name := range names { +// checkManifestPaths fails a manifest that names a skill or file outside its +// directory, or a skill named like the state file. The manifest comes from the +// network, so it must not be able to name ../../.bashrc, and only a broken or +// tampered manifest lists such an entry. The rule is the same on every OS. +// Entries that stay inside their directory but are not portable are left to +// unportableSkill, which skips only that skill. +func checkManifestPaths(m *Manifest) error { + for _, name := range slices.Sorted(maps.Keys(m.Skills)) { if name == "" || name == "." || name == ".." || name == stateFileName || strings.ContainsAny(name, `/\`) { return fmt.Errorf("%w: skill name %q", ErrUnsafeManifestPath, name) } @@ -106,12 +103,6 @@ func checkManifestPaths(ctx context.Context, m *Manifest) error { } } } - for _, name := range names { - if reason := unportableSkill(name, m.Skills[name].Files); reason != "" { - log.Warnf(ctx, "Skipping skill %s: %s", name, reason) - delete(m.Skills, name) - } - } return nil } @@ -139,9 +130,12 @@ func escapesDir(file string) bool { } // unportableSkill returns why a skill that stays inside its directory still -// cannot be installed the same way on every OS, or "" if it can. Files must be -// clean slash-separated paths, so the paths recorded in the install state match -// the ones update walks on disk. +// cannot be installed the same way on every OS, or "" if it can. Install skips +// such a skill and update keeps the installed copy, the way both treat a skill +// that needs a newer CLI. Each file must be a clean slash-separated path, so +// the paths recorded in the install state match the ones update walks on disk. +// Each element is limited to letters, digits, '.', '_' and '-', does not end in +// a dot and is not a Windows device name, and no two files differ only in case. func unportableSkill(name string, files []string) string { if len(name) > maxSkillNameLength || !skillNamePattern().MatchString(name) { return fmt.Sprintf("the name is not at most %d lowercase letters, digits and single hyphens", maxSkillNameLength) @@ -149,20 +143,35 @@ func unportableSkill(name string, files []string) string { if isWindowsDeviceName(name) { return "the name is a reserved Windows device name" } + seen := make(map[string]string, len(files)) for _, file := range files { - if file == "." || path.Clean(file) != file || strings.ContainsAny(file, `\:`) { + if file == "." || path.Clean(file) != file { return fmt.Sprintf("file %q is not a clean slash-separated path", file) } for elem := range strings.SplitSeq(file, "/") { + if !portableElementPattern().MatchString(elem) || strings.HasSuffix(elem, ".") { + return fmt.Sprintf("file %q has a path element that is not portable", file) + } stem, _, _ := strings.Cut(elem, ".") if isWindowsDeviceName(stem) { return fmt.Sprintf("file %q names a reserved Windows device", file) } } + folded := strings.ToLower(file) + if other, ok := seen[folded]; ok { + return fmt.Sprintf("files %q and %q differ only in case", other, file) + } + seen[folded] = file } return "" } +// portableElementPattern is the characters a file path element may use: ones +// that mean the same thing in a file name on every OS. +var portableElementPattern = sync.OnceValue(func() *regexp.Regexp { + return regexp.MustCompile(`^[A-Za-z0-9._-]+$`) +}) + // isWindowsDeviceName reports whether s, in any case, is a name Windows // resolves to a device in every directory. func isWindowsDeviceName(s string) bool { diff --git a/libs/aitools/installer/source_test.go b/libs/aitools/installer/source_test.go index dac0c65c7a9..4c88fed3fb0 100644 --- a/libs/aitools/installer/source_test.go +++ b/libs/aitools/installer/source_test.go @@ -2,12 +2,10 @@ package installer import ( "errors" - "maps" "net/http" "net/http/httptest" "os" "path/filepath" - "slices" "strings" "testing" @@ -104,8 +102,8 @@ func TestFetchManifestSizeCap(t *testing.T) { } // TestCheckManifestPaths covers every skill name and file in a fetched -// manifest becoming a local path: an escape fails the manifest, an entry that -// is not portable drops only its skill. +// manifest becoming a local path: an escape fails the manifest, and an entry +// that is not portable makes unportableSkill skip only its skill. func TestCheckManifestPaths(t *testing.T) { tests := []struct { name string @@ -113,7 +111,7 @@ func TestCheckManifestPaths(t *testing.T) { file string // wantErr is the error after the ErrUnsafeManifestPath prefix. wantErr string - // wantSkipped is true when the skill is dropped from the manifest. + // wantSkipped is true when unportableSkill skips the skill. wantSkipped bool }{ {name: "regular file", skill: "databricks-core", file: "SKILL.md"}, @@ -157,27 +155,37 @@ func TestCheckManifestPaths(t *testing.T) { {name: "file with a colon", skill: "databricks-core", file: "notes:v2.md", wantSkipped: true}, {name: "file named like a device", skill: "databricks-core", file: "references/aux.md", wantSkipped: true}, {name: "directory named like a device", skill: "databricks-core", file: "CON/SKILL.md", wantSkipped: true}, + {name: "file with a question mark", skill: "databricks-core", file: "references/what?.md", wantSkipped: true}, + {name: "file with a control character", skill: "databricks-core", file: "references/a\x1b.md", wantSkipped: true}, + {name: "file with a superscript device name", skill: "databricks-core", file: "COM\u00b9.md", wantSkipped: true}, + {name: "console device file", skill: "databricks-core", file: "CONIN$", wantSkipped: true}, + {name: "device stem with a trailing space", skill: "databricks-core", file: "aux .md", wantSkipped: true}, + {name: "element with a trailing dot", skill: "databricks-core", file: "references./auth.md", wantSkipped: true}, + {name: "file with a trailing dot", skill: "databricks-core", file: "SKILL.md.", wantSkipped: true}, + {name: "dotfile", skill: "databricks-core", file: "references/.gitkeep"}, + {name: "underscore and hyphen in a file", skill: "databricks-core", file: "scripts/run_job-v2.py"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - m := &Manifest{Skills: map[string]SkillMeta{ - tc.skill: {Files: []string{tc.file}}, - "other-skill": {Files: []string{"SKILL.md"}}, - }} - err := checkManifestPaths(t.Context(), m) + m := &Manifest{Skills: map[string]SkillMeta{tc.skill: {Files: []string{tc.file}}}} + err := checkManifestPaths(m) if tc.wantErr != "" { require.ErrorIs(t, err, ErrUnsafeManifestPath) assert.EqualError(t, err, ErrUnsafeManifestPath.Error()+": "+tc.wantErr) return } require.NoError(t, err) - _, kept := m.Skills[tc.skill] - assert.Equal(t, !tc.wantSkipped, kept) - assert.Contains(t, m.Skills, "other-skill") + assert.Equal(t, tc.wantSkipped, unportableSkill(tc.skill, []string{tc.file}) != "") }) } } +func TestUnportableSkillFilesDifferingOnlyInCase(t *testing.T) { + assert.Equal(t, `files "SKILL.md" and "skill.md" differ only in case`, unportableSkill("databricks-core", []string{"SKILL.md", "skill.md"})) + assert.Equal(t, `files "refs/a.md" and "REFS/a.md" differ only in case`, unportableSkill("databricks-core", []string{"refs/a.md", "REFS/a.md"})) + assert.Empty(t, unportableSkill("databricks-core", []string{"SKILL.md", "references/SKILL.md"})) +} + func TestCheckManifestPathsReportsTheFirstSkillInOrder(t *testing.T) { m := &Manifest{Skills: map[string]SkillMeta{ "b-skill": {Files: []string{"../b"}}, @@ -185,7 +193,7 @@ func TestCheckManifestPathsReportsTheFirstSkillInOrder(t *testing.T) { "../c": {Files: []string{"SKILL.md"}}, }} for range 10 { - assert.EqualError(t, checkManifestPaths(t.Context(), m), ErrUnsafeManifestPath.Error()+`: skill name "../c"`) + assert.EqualError(t, checkManifestPaths(m), ErrUnsafeManifestPath.Error()+`: skill name "../c"`) } } @@ -197,14 +205,6 @@ func TestFetchSkillsManifestChecksEverySource(t *testing.T) { }}} _, _, err := FetchSkillsManifestWithFallback(t.Context(), src, "v1", false) require.ErrorIs(t, err, ErrUnsafeManifestPath) - - src = &mockManifestSource{manifest: &Manifest{Skills: map[string]SkillMeta{ - "databricks-core": {Files: []string{"SKILL.md"}}, - "Databricks_Sql": {Files: []string{"SKILL.md"}}, - }}} - manifest, _, err := FetchSkillsManifestWithFallback(t.Context(), src, "v1", false) - require.NoError(t, err) - assert.Equal(t, []string{"databricks-core"}, slices.Collect(maps.Keys(manifest.Skills))) } func TestInstallRejectsStateFileSkillNameWithoutTouchingState(t *testing.T) { @@ -232,3 +232,23 @@ func TestInstallRejectsStateFileSkillNameWithoutTouchingState(t *testing.T) { require.NoError(t, err) assert.Equal(t, string(before), string(after)) } + +func TestInstallSkipsUnportableSkill(t *testing.T) { + tmp := setupTestHome(t) + ctx := cmdio.MockDiscard(t.Context()) + setupFetchMock(t) + t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef) + m := testManifest() + m.Skills["Databricks_Core"] = SkillMeta{Version: "0.1.0", Files: []string{"SKILL.md"}} + agent := testAgent(tmp) + + require.NoError(t, InstallSkillsForAgents(ctx, &mockManifestSource{manifest: m}, []*agents.Agent{agent}, InstallOptions{})) + skillsDir := filepath.Join(tmp, ".databricks", "aitools", "skills") + assert.DirExists(t, filepath.Join(skillsDir, "databricks-sql")) + assert.NoDirExists(t, filepath.Join(skillsDir, "Databricks_Core")) + + err := InstallSkillsForAgents(ctx, &mockManifestSource{manifest: m}, []*agents.Agent{agent}, InstallOptions{SpecificSkills: []string{"Databricks_Core"}}) + skillErr, ok := errors.AsType[*SkillError](err) + require.True(t, ok, "got %v", err) + assert.Equal(t, ReasonUnportableSkill, skillErr.Reason) +} diff --git a/libs/aitools/installer/update.go b/libs/aitools/installer/update.go index 30f1d98ab48..f6f45adf957 100644 --- a/libs/aitools/installer/update.go +++ b/libs/aitools/installer/update.go @@ -144,6 +144,14 @@ func UpdateSkills(ctx context.Context, src ManifestSource, targetAgents []*agent continue } + // Keep the installed copy of a skill this CLI cannot install portably, + // as for one that needs a newer CLI. + if reason := unportableSkill(name, meta.Files); reason != "" { + log.Warnf(ctx, "Skipping skill %q: %s", name, reason) + result.Skipped = append(result.Skipped, name) + continue + } + // Filter experimental skills unless state opted in. if meta.IsExperimental() && !state.IncludeExperimental { log.Debugf(ctx, "Skipping experimental skill %s", name) diff --git a/libs/aitools/installer/update_test.go b/libs/aitools/installer/update_test.go index dad9840ce0a..4e55246e34b 100644 --- a/libs/aitools/installer/update_test.go +++ b/libs/aitools/installer/update_test.go @@ -675,6 +675,39 @@ func TestUpdateSkipsMinCLIVersionSkills(t *testing.T) { assert.Contains(t, logBuf.String(), "requires CLI version 0.300.0") } +// TestUpdateKeepsSkillThatBecomesUnportable checks that a skill whose new +// release lists a file that is not portable is skipped, and its installed copy +// is kept rather than pruned as gone from the release. +func TestUpdateKeepsSkillThatBecomesUnportable(t *testing.T) { + tmp := setupTestHome(t) + ctx := cmdio.MockDiscard(t.Context()) + setupFetchMock(t) + t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef) + + var logBuf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) + ctx = log.NewContext(ctx, logger) + + agent := testAgent(tmp) + require.NoError(t, InstallSkillsForAgents(ctx, &mockManifestSource{manifest: testManifest()}, []*agents.Agent{agent}, InstallOptions{})) + + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.0") + updated := testManifest() + updated.Skills["databricks-sql"] = SkillMeta{Version: "0.2.0", Files: []string{"SKILL.md", "references/aux.md"}} + + result, err := UpdateSkills(ctx, &mockManifestSource{manifest: updated}, []*agents.Agent{agent}, UpdateOptions{}) + require.NoError(t, err) + + assert.Contains(t, result.Skipped, "databricks-sql") + assert.Empty(t, result.Removed) + assert.Contains(t, logBuf.String(), `Skipping skill \"databricks-sql\": file \"references/aux.md\" names a reserved Windows device`) + assert.FileExists(t, filepath.Join(tmp, ".databricks", "aitools", "skills", "databricks-sql", "SKILL.md")) + + state, err := LoadState(filepath.Join(tmp, ".databricks", "aitools", "skills")) + require.NoError(t, err) + assert.Equal(t, "0.1.0", state.Skills["databricks-sql"]) +} + func TestFormatUpdateResultCheckMode(t *testing.T) { result := &UpdateResult{ Updated: []SkillUpdate{ From eb44feabcd4cbc00e6c9d12db9aaaf6f729d0453 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:52:53 +0000 Subject: [PATCH 4/4] aitools: deny only unportable file-name characters, check folder case clashes, add acceptance test Addresses the first three comments of the approving review on #7722: - Replace the [A-Za-z0-9._-] allowlist with a denylist of what is actually unportable: control characters, <>:"\|?*, a trailing dot or space, and Windows device names (now including COM1-3 superscripts, CONIN$, CONOUT$, and stems with trailing spaces). c++.md, logo@2x.png and {{.project_name}}/ install again. - Fold the case of every folder prefix, not just whole paths, and skip a skill that uses one path as both a file and a folder. A file listed twice is no longer reported as a case clash. - Add acceptance/experimental/aitools/skills/install-unsafe-manifest: the escaping manifest's text and --output json error (MANIFEST_INVALID) with no file written, the unportable-skill skip warning, and the --skills error. Co-authored-by: Isaac --- .../install-unsafe-manifest/out.test.toml | 1 + .../skills/install-unsafe-manifest/output.txt | 40 ++++++++++++ .../skills/install-unsafe-manifest/script | 18 ++++++ .../skills/install-unsafe-manifest/test.toml | 50 +++++++++++++++ libs/aitools/installer/source.go | 63 +++++++++++++------ libs/aitools/installer/source_test.go | 33 ++++++++-- 6 files changed, 182 insertions(+), 23 deletions(-) create mode 100644 acceptance/experimental/aitools/skills/install-unsafe-manifest/out.test.toml create mode 100644 acceptance/experimental/aitools/skills/install-unsafe-manifest/output.txt create mode 100644 acceptance/experimental/aitools/skills/install-unsafe-manifest/script create mode 100644 acceptance/experimental/aitools/skills/install-unsafe-manifest/test.toml diff --git a/acceptance/experimental/aitools/skills/install-unsafe-manifest/out.test.toml b/acceptance/experimental/aitools/skills/install-unsafe-manifest/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/experimental/aitools/skills/install-unsafe-manifest/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/experimental/aitools/skills/install-unsafe-manifest/output.txt b/acceptance/experimental/aitools/skills/install-unsafe-manifest/output.txt new file mode 100644 index 00000000000..abe89689fec --- /dev/null +++ b/acceptance/experimental/aitools/skills/install-unsafe-manifest/output.txt @@ -0,0 +1,40 @@ + +=== a manifest listing a file outside its skill directory fails before any file is written + +>>> DATABRICKS_SKILLS_REF=unsafe-ref [CLI] aitools install --skills-only --scope=global --agents=claude-code +Installing Databricks skills for Claude Code... +Using skills version unsafe-ref +Fetching skills manifest... +Error: skills manifest names a path outside the skills directory: skill "test-stable" lists file "../../.bashrc" +No skill files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from https://github.com/databricks/databricks-agent-skills/releases and re-run 'databricks aitools install --scope=global --agents=claude-code --skills-only', and report the manifest from [DATABRICKS_HOST] at https://github.com/databricks/cli/issues + +>>> DATABRICKS_SKILLS_REF=unsafe-ref [CLI] aitools install --skills-only --scope=global --agents=claude-code --output json +{ + "scope": "global", + "agents": [], + "error": "skills manifest names a path outside the skills directory: skill \"test-stable\" lists file \"../../.bashrc\"", + "error_category": "MANIFEST_INVALID", + "recovery": "No skill files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from https://github.com/databricks/databricks-agent-skills/releases and re-run 'databricks aitools install --scope=global --agents=claude-code --skills-only --output json', and report the manifest from [DATABRICKS_HOST] at https://github.com/databricks/cli/issues." +} +no skill files were written + +=== a skill with an unportable name is skipped and the others install + +>>> DATABRICKS_SKILLS_REF=unportable-ref [CLI] aitools install --skills-only --scope=global --agents=claude-code +Installing Databricks skills for Claude Code... +Using skills version unportable-ref +Fetching skills manifest... +Warn: Skipping skill "Test_Unportable": the name is not at most 64 lowercase letters, digits and single hyphens +Installed 1 skill in [TEST_TMP_DIR]/home/.databricks/aitools/skills. +test-stable + +=== naming the unportable skill with --skills fails + +>>> DATABRICKS_SKILLS_REF=unportable-ref [CLI] aitools install --skills-only --scope=global --agents=claude-code --skills=Test_Unportable --output json +{ + "scope": "global", + "agents": [], + "error": "skill \"Test_Unportable\" cannot be installed: the name is not at most 64 lowercase letters, digits and single hyphens", + "error_category": "MANIFEST_INVALID", + "recovery": "No skill files were written. Set DATABRICKS_SKILLS_REF to an earlier release tag from https://github.com/databricks/databricks-agent-skills/releases and re-run 'databricks aitools install --scope=global --agents=claude-code --skills-only --skills=Test_Unportable --output json', and report the manifest from [DATABRICKS_HOST] at https://github.com/databricks/cli/issues." +} diff --git a/acceptance/experimental/aitools/skills/install-unsafe-manifest/script b/acceptance/experimental/aitools/skills/install-unsafe-manifest/script new file mode 100644 index 00000000000..b7bdd38436a --- /dev/null +++ b/acceptance/experimental/aitools/skills/install-unsafe-manifest/script @@ -0,0 +1,18 @@ +# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks. +sethome home + +title "a manifest listing a file outside its skill directory fails before any file is written" +musterr trace DATABRICKS_SKILLS_REF=unsafe-ref $CLI aitools install --skills-only --scope=global --agents=claude-code +musterr trace DATABRICKS_SKILLS_REF=unsafe-ref $CLI aitools install --skills-only --scope=global --agents=claude-code --output json +if [ -e "$HOME/.databricks/aitools/.bashrc" ] || [ -d "$HOME/.databricks/aitools/skills/test-stable" ]; then + echo "unexpected: a skill file was written" +else + echo "no skill files were written" +fi + +title "a skill with an unportable name is skipped and the others install" +trace DATABRICKS_SKILLS_REF=unportable-ref $CLI aitools install --skills-only --scope=global --agents=claude-code +ls "$HOME/.databricks/aitools/skills" + +title "naming the unportable skill with --skills fails" +musterr trace DATABRICKS_SKILLS_REF=unportable-ref $CLI aitools install --skills-only --scope=global --agents=claude-code --skills=Test_Unportable --output json diff --git a/acceptance/experimental/aitools/skills/install-unsafe-manifest/test.toml b/acceptance/experimental/aitools/skills/install-unsafe-manifest/test.toml new file mode 100644 index 00000000000..32220324754 --- /dev/null +++ b/acceptance/experimental/aitools/skills/install-unsafe-manifest/test.toml @@ -0,0 +1,50 @@ +# Mock server replaces raw.githubusercontent.com for manifest + skill files. +Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST" + +Ignore = [ + "home", +] + +# A manifest that lists a file outside its skill directory. +[[Server]] +Pattern = "GET /unsafe-ref/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-01-01T00:00:00Z", + "skills": { + "test-stable": { + "version": "1.0.0", + "files": ["SKILL.md", "../../.bashrc"], + "repo_dir": "skills" + } + } +} +''' + +# A manifest with one portable skill and one whose name is not in the Agent +# Skills format. +[[Server]] +Pattern = "GET /unportable-ref/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-01-01T00:00:00Z", + "skills": { + "test-stable": { + "version": "1.0.0", + "files": ["SKILL.md"], + "repo_dir": "skills" + }, + "Test_Unportable": { + "version": "1.0.0", + "files": ["SKILL.md"], + "repo_dir": "skills" + } + } +} +''' + +[[Server]] +Pattern = "GET /unportable-ref/skills/test-stable/SKILL.md" +Response.Body = "test-stable" diff --git a/libs/aitools/installer/source.go b/libs/aitools/installer/source.go index 1a408e81369..0cc06e1fa62 100644 --- a/libs/aitools/installer/source.go +++ b/libs/aitools/installer/source.go @@ -134,8 +134,10 @@ func escapesDir(file string) bool { // such a skill and update keeps the installed copy, the way both treat a skill // that needs a newer CLI. Each file must be a clean slash-separated path, so // the paths recorded in the install state match the ones update walks on disk. -// Each element is limited to letters, digits, '.', '_' and '-', does not end in -// a dot and is not a Windows device name, and no two files differ only in case. +// No path element may use a character Windows forbids in a file name, end in a +// dot or a space, or be a Windows device name. No two paths, folders included, +// may differ only in case, and no path may be both a file and a folder: either +// would merge or collide on a case-insensitive filesystem. func unportableSkill(name string, files []string) string { if len(name) > maxSkillNameLength || !skillNamePattern().MatchString(name) { return fmt.Sprintf("the name is not at most %d lowercase letters, digits and single hyphens", maxSkillNameLength) @@ -143,44 +145,67 @@ func unportableSkill(name string, files []string) string { if isWindowsDeviceName(name) { return "the name is a reserved Windows device name" } - seen := make(map[string]string, len(files)) + type seenPath struct { + path string + dir bool + } + seen := make(map[string]seenPath, len(files)) for _, file := range files { if file == "." || path.Clean(file) != file { return fmt.Sprintf("file %q is not a clean slash-separated path", file) } - for elem := range strings.SplitSeq(file, "/") { - if !portableElementPattern().MatchString(elem) || strings.HasSuffix(elem, ".") { + elems := strings.Split(file, "/") + for i, elem := range elems { + if strings.ContainsFunc(elem, isUnportableRune) || strings.HasSuffix(elem, ".") || strings.HasSuffix(elem, " ") { return fmt.Sprintf("file %q has a path element that is not portable", file) } stem, _, _ := strings.Cut(elem, ".") - if isWindowsDeviceName(stem) { + if isWindowsDeviceName(strings.TrimRight(stem, " ")) { return fmt.Sprintf("file %q names a reserved Windows device", file) } + p := strings.Join(elems[:i+1], "/") + dir := i < len(elems)-1 + folded := strings.ToLower(p) + prev, ok := seen[folded] + switch { + case !ok: + seen[folded] = seenPath{path: p, dir: dir} + case prev.path != p: + return fmt.Sprintf("paths %q and %q differ only in case", prev.path, p) + case prev.dir != dir: + return fmt.Sprintf("path %q is both a file and a folder", p) + } } - folded := strings.ToLower(file) - if other, ok := seen[folded]; ok { - return fmt.Sprintf("files %q and %q differ only in case", other, file) - } - seen[folded] = file } return "" } -// portableElementPattern is the characters a file path element may use: ones -// that mean the same thing in a file name on every OS. -var portableElementPattern = sync.OnceValue(func() *regexp.Regexp { - return regexp.MustCompile(`^[A-Za-z0-9._-]+$`) -}) +// isUnportableRune reports whether r cannot appear in a file name on Windows: +// a control character or one of <>:"/\|?*. +func isUnportableRune(r rune) bool { + return r < 0x20 || r == 0x7f || strings.ContainsRune(`<>:"\|?*`, r) +} // isWindowsDeviceName reports whether s, in any case, is a name Windows -// resolves to a device in every directory. +// resolves to a device in every directory, as in Go's internal isReservedName. func isWindowsDeviceName(s string) bool { s = strings.ToLower(s) switch s { - case "con", "prn", "aux", "nul": + case "con", "prn", "aux", "nul", "conin$", "conout$": return true } - return len(s) == 4 && (strings.HasPrefix(s, "com") || strings.HasPrefix(s, "lpt")) && s[3] >= '1' && s[3] <= '9' + rest, ok := strings.CutPrefix(s, "com") + if !ok { + rest, ok = strings.CutPrefix(s, "lpt") + } + if !ok { + return false + } + switch rest { + case "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u00b9", "\u00b2", "\u00b3": + return true + } + return false } // normalizeManifest stamps SourceName and defaults RepoDir for older manifests. diff --git a/libs/aitools/installer/source_test.go b/libs/aitools/installer/source_test.go index 4c88fed3fb0..dbe96be9a54 100644 --- a/libs/aitools/installer/source_test.go +++ b/libs/aitools/installer/source_test.go @@ -164,6 +164,15 @@ func TestCheckManifestPaths(t *testing.T) { {name: "file with a trailing dot", skill: "databricks-core", file: "SKILL.md.", wantSkipped: true}, {name: "dotfile", skill: "databricks-core", file: "references/.gitkeep"}, {name: "underscore and hyphen in a file", skill: "databricks-core", file: "scripts/run_job-v2.py"}, + {name: "file with plus signs", skill: "databricks-core", file: "references/c++.md"}, + {name: "file with an at sign", skill: "databricks-core", file: "assets/logo@2x.png"}, + {name: "template folder with braces", skill: "databricks-core", file: "{{.project_name}}/databricks.yml"}, + {name: "file with an inner space", skill: "databricks-core", file: "references/getting started.md"}, + {name: "file with parentheses, comma and tilde", skill: "databricks-core", file: "notes (v2),~draft.md"}, + {name: "non-ASCII file", skill: "databricks-core", file: "references/caf\u00e9.md"}, + {name: "element with a trailing space", skill: "databricks-core", file: "references /auth.md", wantSkipped: true}, + {name: "file with a pipe", skill: "databricks-core", file: "a|b.md", wantSkipped: true}, + {name: "file with a quote", skill: "databricks-core", file: `a"b.md`, wantSkipped: true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -180,10 +189,26 @@ func TestCheckManifestPaths(t *testing.T) { } } -func TestUnportableSkillFilesDifferingOnlyInCase(t *testing.T) { - assert.Equal(t, `files "SKILL.md" and "skill.md" differ only in case`, unportableSkill("databricks-core", []string{"SKILL.md", "skill.md"})) - assert.Equal(t, `files "refs/a.md" and "REFS/a.md" differ only in case`, unportableSkill("databricks-core", []string{"refs/a.md", "REFS/a.md"})) - assert.Empty(t, unportableSkill("databricks-core", []string{"SKILL.md", "references/SKILL.md"})) +func TestUnportableSkillPathClashes(t *testing.T) { + tests := []struct { + name string + files []string + want string + }{ + {name: "files differing only in case", files: []string{"SKILL.md", "skill.md"}, want: `paths "SKILL.md" and "skill.md" differ only in case`}, + {name: "folders differing only in case", files: []string{"refs/a.md", "REFS/b.md"}, want: `paths "refs" and "REFS" differ only in case`}, + {name: "file and folder differing only in case", files: []string{"Refs", "refs/a.md"}, want: `paths "Refs" and "refs" differ only in case`}, + {name: "path used as a file and a folder", files: []string{"scripts", "scripts/run.py"}, want: `path "scripts" is both a file and a folder`}, + {name: "folder then file of the same path", files: []string{"scripts/run.py", "scripts"}, want: `path "scripts" is both a file and a folder`}, + {name: "same file listed twice", files: []string{"SKILL.md", "SKILL.md"}}, + {name: "same name in different folders", files: []string{"SKILL.md", "references/SKILL.md"}}, + {name: "files sharing a folder", files: []string{"refs/a.md", "refs/b.md"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, unportableSkill("databricks-core", tc.files)) + }) + } } func TestCheckManifestPathsReportsTheFirstSkillInOrder(t *testing.T) {