From c826639f38ccfa20484a2fab48f6d389b8224399 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Thu, 12 Nov 2026 13:25:46 +0000 Subject: [PATCH 01/43] Log AitoolsMcpEvent from aitools mcp and the serve proxy The MCP commands and the proxy now buffer an AitoolsMcpEvent for the telemetry upload at exit: one result per (service, agent) for add and remove, one per service for login and logout, and PROXY_STARTED and SIGN_IN_PROMPTED from serve. Events carry enums and counts only; no host, profile, workspace ID, custom service or schema name, path or error text. The event mirrors the lumberjack schema for aitools MCP, which is not merged yet. The enum values for the MCP error categories are added next to the install ones, and the commands use them in place of local constants. Profile, sign-in and list failures that had no typed category now report NO_PROFILE, PROFILE_REQUIRED, MCP_PERMISSION_DENIED, MCP_LOCATION_NOT_FOUND and MCP_NAME_CONFLICT. The fake /telemetry-ext now reports the batch size, as the server does. Before, a batch of two or more events was resent by the CLI's retry. Co-authored-by: Isaac --- .../aitools/telemetry/mcp-add/.databrickscfg | 3 + .../aitools/telemetry/mcp-add/out.test.toml | 1 + .../cmd/aitools/telemetry/mcp-add/output.txt | 90 +++++++ .../cmd/aitools/telemetry/mcp-add/script | 29 +++ .../telemetry/mcp-remove/.databrickscfg | 7 + .../telemetry/mcp-remove/out.test.toml | 1 + .../aitools/telemetry/mcp-remove/output.txt | 70 ++++++ .../cmd/aitools/telemetry/mcp-remove/script | 34 +++ .../telemetry/mcp-serve/.databrickscfg | 3 + .../aitools/telemetry/mcp-serve/out.test.toml | 1 + .../aitools/telemetry/mcp-serve/output.txt | 38 +++ .../telemetry/mcp-serve/requests-after.ndjson | 2 + .../mcp-serve/requests-authenticate.ndjson | 1 + .../telemetry/mcp-serve/requests-list.ndjson | 3 + .../cmd/aitools/telemetry/mcp-serve/script | 40 +++ .../telemetry/mcp-sign-in/.databrickscfg | 3 + .../telemetry/mcp-sign-in/out.test.toml | 1 + .../aitools/telemetry/mcp-sign-in/output.txt | 98 ++++++++ .../cmd/aitools/telemetry/mcp-sign-in/script | 51 ++++ cmd/aitools/mcp.go | 21 +- cmd/aitools/mcp_add.go | 32 ++- cmd/aitools/mcp_add_test.go | 28 +-- cmd/aitools/mcp_login.go | 40 ++- cmd/aitools/mcp_login_test.go | 4 +- cmd/aitools/mcp_remove.go | 18 +- cmd/aitools/mcp_remove_test.go | 8 +- cmd/aitools/mcp_serve.go | 10 + cmd/aitools/mcp_telemetry.go | 228 +++++++++++++++++ cmd/aitools/mcp_telemetry_test.go | 235 ++++++++++++++++++ cmd/aitools/mcp_test.go | 10 +- libs/aitools/mcp/proxy/bridge_test.go | 2 +- libs/aitools/mcp/proxy/challenge.go | 17 +- libs/aitools/mcp/proxy/challenge_test.go | 52 +++- libs/telemetry/protos/aitools_install.go | 17 ++ libs/telemetry/protos/aitools_mcp.go | 94 +++++++ libs/telemetry/protos/aitools_mcp_test.go | 57 +++++ libs/telemetry/protos/frontend_log.go | 1 + 37 files changed, 1287 insertions(+), 63 deletions(-) create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/script create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/script create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/script create mode 100644 acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-sign-in/script create mode 100644 cmd/aitools/mcp_telemetry.go create mode 100644 cmd/aitools/mcp_telemetry_test.go create mode 100644 libs/telemetry/protos/aitools_mcp.go create mode 100644 libs/telemetry/protos/aitools_mcp_test.go diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/output.txt b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt new file mode 100644 index 00000000000..b7ea889e8b8 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt @@ -0,0 +1,90 @@ + +=== A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "PENDING" + }, + { + "service": "GITHUB", + "agent": "CODEX", + "outcome": "CHANGED", + "sign_in_state": "PENDING" + } + ], + "bound_profile_count": 1 +} + +=== The same servers again are re-asserted: UNCHANGED +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "PENDING" + }, + { + "service": "GITHUB", + "agent": "CODEX", + "outcome": "UNCHANGED", + "sign_in_state": "PENDING" + } + ], + "bound_profile_count": 1 +} + +=== An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID + +Exit code: 1 +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "FAILED", + "error_category": "AGENT_CONFIG_INVALID", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 1 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/script b/acceptance/cmd/aitools/telemetry/mcp-add/script new file mode 100644 index 00000000000..83d6b223112 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/script @@ -0,0 +1,29 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" + rm -f "$OUT_REQUESTS" + fi +} + +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING" +$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +mcp_events + +title "The same servers again are re-asserted: UNCHANGED" +$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +mcp_events + +title "An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID" +printf 'not = [valid\n' > home/.codex/config.toml +errcode $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json > /dev/null +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg new file mode 100644 index 00000000000..d64cd1da28f --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg @@ -0,0 +1,7 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood + +[dogfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt new file mode 100644 index 00000000000..6ab420b2562 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -0,0 +1,70 @@ + +=== Installed for two profiles: the bound profile count is 2 after the second add +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 1 +} +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 2 +} + +=== remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 1 +} + +=== An entry changed outside this CLI is left in place: SKIPPED +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "SKIPPED" + } + ], + "bound_profile_count": 1 +} + +=== An entry already gone from the agent config is SKIPPED, and the state record goes +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "SKIPPED" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script new file mode 100644 index 00000000000..b25e4b4a31a --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -0,0 +1,34 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" + rm -f "$OUT_REQUESTS" + fi +} + +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "Installed for two profiles: the bound profile count is 2 after the second add" +$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +mcp_events +$CLI aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +mcp_events + +title "remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound" +$CLI aitools mcp remove --profile dogfood --output json > /dev/null +mcp_events + +title "An entry changed outside this CLI is left in place: SKIPPED" +jq '.mcpServers["genie_one_mcp-logfood"].command = "/usr/bin/other"' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json +$CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +mcp_events + +title "An entry already gone from the agent config is SKIPPED, and the state record goes" +jq 'del(.mcpServers["genie_one_mcp-logfood"])' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json +$CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt new file mode 100644 index 00000000000..08586c94def --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -0,0 +1,38 @@ + +=== serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GENIE_ONE" + } + ], + "bound_profile_count": 0 +} + +=== A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with + +=== authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GITHUB" + } + ], + "bound_profile_count": 0 +} +{ + "calling_agent": "CLAUDE_CODE", + "operation": "SIGN_IN_PROMPTED", + "servers": [ + { + "service": "GITHUB", + "error_category": "SIGN_IN_REQUIRED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson new file mode 100644 index 00000000000..57d5e4a4066 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson @@ -0,0 +1,2 @@ +{"jsonrpc":"2.0","id":4,"method":"tools/list"} +{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"echo","arguments":{"text":"hello"}}} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson new file mode 100644 index 00000000000..cc792544a79 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"authenticate","arguments":{}}} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson new file mode 100644 index 00000000000..84c14278376 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson @@ -0,0 +1,3 @@ +{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"test-client","version":"0.0.1"}}} +{"jsonrpc":"2.0","method":"notifications/initialized"} +{"jsonrpc":"2.0","id":2,"method":"tools/list"} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script new file mode 100644 index 00000000000..fdceb646167 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -0,0 +1,40 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# No browser: the proxy prints the sign-in URL and does not open anything. +export BROWSER=none +export RETRY_MAX_ATTEMPTS=60 +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" + rm -f "$OUT_REQUESTS" + fi +} + +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes" +$CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null 2> /dev/null +mcp_events + +title "A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with" +errcode $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null 2> /dev/null +mcp_events + +title "authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait" +{ + cat requests-list.ndjson + retry --until '"id":2' cat serve.stdout.txt > /dev/null + cat requests-authenticate.ndjson + retry --until "Waiting for sign-in" cat serve.stderr.txt > /dev/null + $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null + retry --until '"id":3' cat serve.stdout.txt > /dev/null + cat requests-after.ndjson +} | $CLI aitools mcp serve --profile logfood --service system.ai.github --agent claude-code > serve.stdout.txt 2> serve.stderr.txt +mcp_events +rm serve.stdout.txt serve.stderr.txt diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt new file mode 100644 index 00000000000..d2b56d441f2 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt @@ -0,0 +1,98 @@ + +=== login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED +system.ai.genie_one_mcp does not need sign-in. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GENIE_ONE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 0 +} + +=== login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} + +=== login while already signed in: UNCHANGED +Already signed in to system.ai.github on profile logfood. +Public repositories only until a repository owner installs https://github.com/apps/databricks-github-connector. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "UNCHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} + +=== logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED +Signed out of system.ai.github on profile logfood. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_OUT", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} +Not signed in to system.ai.github on profile logfood. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_OUT", + "servers": [ + { + "service": "GITHUB", + "outcome": "UNCHANGED" + } + ], + "bound_profile_count": 0 +} + +=== add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 1 +} +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 1 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script new file mode 100644 index 00000000000..943af911b86 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script @@ -0,0 +1,51 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# No browser: the CLI prints the sign-in URL and polls. +export BROWSER=none +export RETRY_MAX_ATTEMPTS=60 +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" + rm -f "$OUT_REQUESTS" + fi +} + +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED" +$CLI aitools mcp login system.ai.genie_one_mcp --profile logfood > /dev/null +mcp_events + +title "login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE" +$CLI aitools mcp login system.ai.github --profile logfood > login.txt 2>&1 & +PID=$! +retry --until "Waiting for sign-in" cat login.txt > /dev/null +$CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +wait $PID +mcp_events +rm login.txt + +title "login while already signed in: UNCHANGED" +$CLI aitools mcp login system.ai.github --profile logfood > /dev/null +mcp_events + +title "logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED" +$CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +mcp_events +$CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +mcp_events + +title "add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE" +$CLI aitools mcp add --profile logfood --names system.ai.github --agents claude-code --login > add.txt 2>&1 & +PID=$! +retry --until "Waiting for sign-in" cat add.txt > /dev/null +$CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +wait $PID +mcp_events +rm add.txt diff --git a/cmd/aitools/mcp.go b/cmd/aitools/mcp.go index ee8dd76ec85..edc6ed34b4b 100644 --- a/cmd/aitools/mcp.go +++ b/cmd/aitools/mcp.go @@ -17,19 +17,6 @@ import ( "github.com/spf13/cobra" ) -// Error categories that the telemetry enum does not define yet. The JSON output reports them by name. The -// binding categories come from mcp.BindCategory, which has the same names. -const ( - categoryAgentConfigInvalid protos.AitoolsErrorCategory = "AGENT_CONFIG_INVALID" - categoryNameConflict protos.AitoolsErrorCategory = "MCP_NAME_CONFLICT" - categoryServiceNotFound protos.AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" - categoryNoProfile protos.AitoolsErrorCategory = "NO_PROFILE" - categoryProfileRequired protos.AitoolsErrorCategory = "PROFILE_REQUIRED" - categoryPermissionDenied protos.AitoolsErrorCategory = "MCP_PERMISSION_DENIED" - categoryLocationNotFound protos.AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" - categorySignInTimeout protos.AitoolsErrorCategory = "SIGN_IN_TIMEOUT" -) - // NewMcpCmd returns the `aitools mcp` group. It stays hidden until the whole MCP // stack has landed, so a release never ships `add` without `serve`. func NewMcpCmd() *cobra.Command { @@ -84,9 +71,9 @@ func errorCategory(err error) protos.AitoolsErrorCategory { } switch { case errors.Is(err, agentconfig.ErrConfigInvalid): - return categoryAgentConfigInvalid + return protos.AitoolsErrorCategoryAgentConfigInvalid case errors.Is(err, agentconfig.ErrNotOwned): - return categoryNameConflict + return protos.AitoolsErrorCategoryMCPNameConflict } return classifyInstallError(err) } @@ -116,11 +103,11 @@ func resolveProfile(ctx context.Context, flagValue string) (string, error) { } switch len(profiles) { case 0: - return "", failure(categoryNoProfile, "no workspace profiles in %s. Run: databricks auth login --host --profile ", filepath.ToSlash(path)) + return "", failure(protos.AitoolsErrorCategoryNoProfile, "no workspace profiles in %s. Run: databricks auth login --host --profile ", filepath.ToSlash(path)) case 1: return profiles[0].Name, nil } - return "", failureWithCause(categoryProfileRequired, &profileChoice{profiles: profiles}, "several workspace profiles exist; pass --profile. Found: %s", strings.Join(profiles.Names(), ", ")) + return "", failureWithCause(protos.AitoolsErrorCategoryProfileRequired, &profileChoice{profiles: profiles}, "several workspace profiles exist; pass --profile. Found: %s", strings.Join(profiles.Names(), ", ")) } // mcpWorkspaceProfile reports whether auto-detect may pick p. It accepts the profiles that --profile accepts: a diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index e727ae14235..d73ea78847c 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -20,6 +20,7 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/browser" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" @@ -188,6 +189,11 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), Servers: []addServerOutput{}, } runErr := runMcpAdd(ctx, cmd, f, jsonMode, out) + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationAdd, + Servers: addServerResults(out.Servers), + }) if !jsonMode { return runErr } @@ -375,6 +381,8 @@ func runMcpAdd(ctx context.Context, cmd *cobra.Command, f mcpAddFlags, jsonMode return err } } + // The bound profile authenticates the exit-time telemetry upload. + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), w.Config)) for i := range plans { plans[i].service = picked[i] } @@ -616,9 +624,9 @@ func describeListError(profileName string, err error) error { var category protos.AitoolsErrorCategory switch listErr.Kind { case mcp.ListErrLocationNotFound: - category = categoryLocationNotFound + category = protos.AitoolsErrorCategoryMCPLocationNotFound case mcp.ListErrPermissionDenied: - category = categoryPermissionDenied + category = protos.AitoolsErrorCategoryMCPPermissionDenied case mcp.ListErrRateLimited: category = protos.AitoolsErrorCategoryRateLimited default: @@ -807,13 +815,13 @@ func pickServices(location string, names []string, services []mcp.Service) ([]mc i := slices.IndexFunc(services, func(s mcp.Service) bool { return strings.EqualFold(s.FullName, name) }) if i < 0 { if len(services) == 0 { - return nil, failure(categoryServiceNotFound, "%s is not an MCP service in %s, which has none", name, location) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "%s is not an MCP service in %s, which has none", name, location) } available := make([]string, 0, len(services)) for _, s := range services { available = append(available, s.FullName) } - return nil, failure(categoryServiceNotFound, "%s is not an MCP service in %s. Available: %s", name, location, strings.Join(available, ", ")) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "%s is not an MCP service in %s. Available: %s", name, location, strings.Join(available, ", ")) } picked = append(picked, services[i]) } @@ -961,7 +969,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st case 1: stored = owned[0] default: - return "", nil, "", failure(categoryNameConflict, "%s is installed on profile %s under more than one name in the agents (%s); remove the extra entries", fullName, b.profile, strings.Join(owned, ", ")) + return "", nil, "", failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s is installed on profile %s under more than one name in the agents (%s); remove the extra entries", fullName, b.profile, strings.Join(owned, ", ")) } } @@ -977,7 +985,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st return "", nil, "", failure(protos.AitoolsErrorCategoryInvalidFlags, "%s is already installed on profile %s as %s; remove it before using --name %s", fullName, b.profile, stored, in.nameFlag) } if stateHolds(state, stored, entry) { - return "", nil, "", failure(categoryNameConflict, "%s is installed as %s, but another server in the state holds %s; remove that entry first", fullName, stored, stored) + return "", nil, "", failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s is installed as %s, but another server in the state holds %s; remove that entry first", fullName, stored, stored) } failed, err := nameFailures(ctx, in, fullName, stored, true) return stored, failed, "", err @@ -1087,7 +1095,7 @@ func holderConflict(ctx context.Context, in planInput, a *agents.Agent, fullName if in.force { return nil, nil } - return failure(categoryNameConflict, "%s already has a server named %s that this CLI did not write; pass --force to replace it", a.DisplayName, name), nil + return failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s already has a server named %s that this CLI did not write; pass --force to replace it", a.DisplayName, name), nil } serves, err := mcp.EntryServes(ctx, entry, fullName, in.binding.profile, in.binding.configFile) if err != nil { @@ -1097,7 +1105,7 @@ func holderConflict(ctx context.Context, in planInput, a *agents.Agent, fullName case serves: return nil, nil case stored: - return failure(categoryNameConflict, "%s is installed as %s, but %s holds %s for another server; remove that entry first", fullName, name, a.Name, name), nil + return failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s is installed as %s, but %s holds %s for another server; remove that entry first", fullName, name, a.Name, name), nil default: return failure(protos.AitoolsErrorCategoryInvalidFlags, "--name %s is used by another server in %s; pick a different name", name, a.Name), nil } @@ -1264,11 +1272,17 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp } } var state string + var alreadySignedIn bool url, err := mcp.SignInURL(w.Config.Host, plans[i].service.FullName, req.workspaceID) if err == nil { - state, _, _, err = signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) + state, alreadySignedIn, _, err = signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) } server.SignIn = &addSignInOutput{State: state} + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{signInServerResult(server.Service, state, alreadySignedIn, err)}, + }) if err != nil { server.SignIn.ErrorCategory = errorCategory(err) server.SignIn.Error = err.Error() diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index 59fe15408db..1ccd09a5e17 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -207,7 +207,7 @@ func TestPickServices(t *testing.T) { _, err = pickServices("system.ai", []string{"system.ai.foo"}, services) require.Error(t, err) assert.Equal(t, "system.ai.foo is not an MCP service in system.ai. Available: system.ai.genie_one_mcp, system.ai.github", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) } func TestResolveMCPAgents(t *testing.T) { @@ -257,7 +257,7 @@ func TestDescribeListErrorNamesProfileAndKeepsCause(t *testing.T) { err := describeListError("logfood", listErr) assert.Equal(t, "profile logfood: "+listErr.Error(), err.Error()) - assert.Equal(t, categoryPermissionDenied, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPPermissionDenied, errorCategory(err)) got, ok := errors.AsType[*mcp.ListError](err) require.True(t, ok) @@ -268,7 +268,7 @@ func TestDescribeListErrorNamesProfileAndKeepsCause(t *testing.T) { assert.Equal(t, protos.AitoolsErrorCategoryRateLimited, errorCategory(rateLimited)) notFound := describeListError("logfood", &mcp.ListError{Kind: mcp.ListErrLocationNotFound, Location: "main.tools", Err: cause}) - assert.Equal(t, categoryLocationNotFound, errorCategory(notFound)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPLocationNotFound, errorCategory(notFound)) plain := errors.New("network down") assert.Equal(t, plain, describeListError("logfood", plain)) @@ -553,7 +553,7 @@ func TestPlanAddsNaming(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(openCode, "opencode.json"), []byte(`{"mcp":{"servers":{"two":{"type":"local","command":["databricks","aitools","mcp","serve","--profile","logfood","--service","system.ai.github","--agent","opencode"]}}}}`), 0o600)) _, err := planNames(t, "", defaultBinding(t, "logfood"), &mcp.State{}, []string{"system.ai.github"}, []*agents.Agent{claude, agents.ByName("opencode")}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) }) t.Run("OpenCode entry with its variables in environment is reused", func(t *testing.T) { @@ -596,7 +596,7 @@ func TestPlanAddsNaming(t *testing.T) { require.Len(t, plans, 1) assert.Equal(t, "github-logfood", plans[0].name) require.Contains(t, plans[0].failed, claude) - assert.Equal(t, categoryNameConflict, errorCategory(plans[0].failed[claude])) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(plans[0].failed[claude])) }) t.Run("planning reads the agent entries it is given, not the files", func(t *testing.T) { @@ -885,7 +885,7 @@ func TestAgentFailureErrorWhenAServiceReachedNoAgent(t *testing.T) { } func TestErrorCategoryMapsTypedFailures(t *testing.T) { - assert.Equal(t, categoryNameConflict, + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(fmt.Errorf("claude-code: server %q: %w", "github", agentconfig.ErrNotOwned))) assert.Equal(t, protos.AitoolsErrorCategoryTimeout, errorCategory(fmt.Errorf("list: %w", context.DeadlineExceeded))) assert.Equal(t, protos.AitoolsErrorCategoryPermissionDenied, errorCategory(&fs.PathError{Op: "open", Path: "x", Err: fs.ErrPermission})) @@ -1077,7 +1077,7 @@ func TestMcpAddRefusesStoredNameHeldByOtherBinding(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github", "--agents", "claude-code") require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.ErrorContains(t, err, "github-logfood") got, err := os.ReadFile(claudeFile) require.NoError(t, err) @@ -1253,7 +1253,7 @@ func TestPlanAddsRefusesRecoveredNameHeldByAnotherServer(t *testing.T) { _, err := planNames(t, "", defaultBinding(t, "logfood"), state, []string{"system.ai.github"}, []*agents.Agent{agents.ByName("claude-code")}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.Equal(t, "system.ai.github is installed as github-logfood, but another server in the state holds github-logfood; remove that entry first", err.Error()) } @@ -1313,7 +1313,7 @@ func TestMcpAddRefusesUnmanagedEntryBeforeAnyRequest(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github", "--agents", "claude-code") require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.ErrorContains(t, err, "Claude Code already has a server named github-logfood that this CLI did not write; pass --force to replace it") got, err := os.ReadFile(claudeFile) require.NoError(t, err) @@ -1495,14 +1495,14 @@ func TestPlanAddsDetectedAgentConflictFailsOnlyThatAgent(t *testing.T) { require.Len(t, plans, 1) assert.Equal(t, "github-logfood", plans[0].name) require.Contains(t, plans[0].failed, claude) - assert.Equal(t, categoryNameConflict, errorCategory(plans[0].failed[claude])) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(plans[0].failed[claude])) assert.NotContains(t, plans[0].failed, codex) assert.NoError(t, planningRefusals(plans, targets, nil)) in.explicit = true _, err = planAdds(ctx, in, state, []string{"system.ai.github"}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) onlyClaude := []*agents.Agent{claude} in = planInput{binding: defaultBinding(t, "logfood"), targets: onlyClaude, held: readHeld(ctx, onlyClaude)} @@ -1510,7 +1510,7 @@ func TestPlanAddsDetectedAgentConflictFailsOnlyThatAgent(t *testing.T) { require.NoError(t, err) refusal := planningRefusals(plans, onlyClaude, nil) require.Error(t, refusal) - assert.Equal(t, categoryNameConflict, errorCategory(refusal)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(refusal)) } // TestPlanAddsRefusedAgentConflictDoesNotRefuseRun checks that with --agents, a name conflict in an agent that is refused, and so @@ -1547,7 +1547,7 @@ func TestPlanAddsUnmanagedEntryIsReplacedOnlyWithForce(t *testing.T) { in := planInput{binding: defaultBinding(t, "logfood"), targets: targets, held: held, explicit: true} _, err := planAdds(ctx, in, state, []string{"system.ai.github"}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) in.force = true plans, err := planAdds(ctx, in, state, []string{"system.ai.github"}) @@ -1560,7 +1560,7 @@ func TestPickServicesNamesSchemaWithNoServices(t *testing.T) { _, err := pickServices("system.ai", []string{"system.ai.github"}, nil) require.Error(t, err) assert.Equal(t, "system.ai.github is not an MCP service in system.ai, which has none", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) } // TestInstallServerWritesOnlyAgentsInstalledForMCP checks that an agent named with --agents is written when it counts as installed for MCP, and that diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index c2f2da23d58..9307ed94013 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -12,10 +12,12 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/browser" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" ) @@ -45,6 +47,11 @@ type mcpSignInOutput struct { URL string `json:"url,omitempty"` ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` Error string `json:"error,omitempty"` + + // cfg is the bound profile's config, which authenticates the exit-time telemetry upload. + cfg *config.Config + // alreadySignedIn is set by login when the caller was signed in before it ran. + alreadySignedIn bool } // mcpSignInAction runs login or logout against the profile's client and records the result in out. pinnedFile is the @@ -65,7 +72,7 @@ sign-in to complete. Without a browser, open the printed URL yourself. databricks aitools mcp login system.ai.github --profile logfood`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runMcpSignInCommand(cmd, args[0], mcpLogin) + return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } } @@ -83,19 +90,23 @@ stays in place. Running it when you are not signed in is not an error. databricks aitools mcp logout system.ai.github --profile logfood`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runMcpSignInCommand(cmd, args[0], mcpLogout) + return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } } // runMcpSignInCommand resolves the profile for service and runs action. In JSON mode the document is // the only stdout output; progress goes to stderr. -func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAction) error { +func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAction, operation protos.AitoolsMcpOperation) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) out := &mcpSignInOutput{Service: service} runErr := runMcpSignIn(ctx, cmd.ErrOrStderr(), profileFlag(cmd), service, action, out) + if out.cfg != nil { + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), out.cfg)) + } + logMcpEvent(ctx, signInEvent(operation, out, runErr)) if !jsonMode { return runErr } @@ -160,15 +171,31 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri return err } out.Host = w.Config.Host + out.cfg = w.Config return action(ctx, errOut, w, out, pinnedFile) } +// signInEvent is the SIGN_IN or SIGN_OUT event of one login or logout. +func signInEvent(operation protos.AitoolsMcpOperation, out *mcpSignInOutput, err error) protos.AitoolsMcpEvent { + var r protos.AitoolsMcpServerResult + if operation == protos.AitoolsMcpOperationSignIn { + r = signInServerResult(out.Service, out.State, out.alreadySignedIn, err) + } else { + r = signOutServerResult(out.Service, out.State, err) + } + return protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: operation, + Servers: []protos.AitoolsMcpServerResult{r}, + } +} + // serviceFor returns the service with its sign-in requirement, as the profile sees it. A service the profile cannot see // is an error, so that a misspelled name is reported rather than treated as a service with no sign-in. func serviceFor(ctx context.Context, w *databricks.WorkspaceClient, profileName, pinnedFile, service string) (mcp.Service, error) { svc, err := mcp.GetService(ctx, w, service) if mcp.IsNotFound(err) { - return mcp.Service{}, failureWithCause(categoryServiceNotFound, err, "%s is not an MCP service visible to profile %s", service, profileName) + return mcp.Service{}, failureWithCause(protos.AitoolsErrorCategoryMCPServiceNotFound, err, "%s is not an MCP service visible to profile %s", service, profileName) } if err != nil { return mcp.Service{}, describeRequestError(profileName, pinnedFile, err) @@ -199,8 +226,9 @@ func mcpLogin(ctx context.Context, errOut io.Writer, w *databricks.WorkspaceClie if err != nil { return err } - state, _, urlShown, err := signIn(ctx, errOut, w, pinnedFile, url, svc, signInOptions{Profile: out.Profile, Open: browser.OpenOutputToStderr}) + state, alreadySignedIn, urlShown, err := signIn(ctx, errOut, w, pinnedFile, url, svc, signInOptions{Profile: out.Profile, Open: browser.OpenOutputToStderr}) out.State = state + out.alreadySignedIn = alreadySignedIn // The URL is named when it was printed. A sign-in that failed before its page is printed names none. if urlShown { out.URL = url @@ -287,7 +315,7 @@ func signIn(ctx context.Context, errOut io.Writer, w *databricks.WorkspaceClient timeout, timedOut := errors.AsType[*mcp.SignInTimeoutError](err) switch { case timedOut: - return signInStateTimeout, false, urlShown, failureWithCause(categorySignInTimeout, err, + return signInStateTimeout, false, urlShown, failureWithCause(protos.AitoolsErrorCategorySignInTimeout, err, "sign-in to %s did not complete in %g minutes%s. Finish at %s then retry: %s", svc.FullName, signInWaitTimeout.Minutes(), lastCheckText(timeout), url, mcpLoginHint(svc.FullName, profileName, pinnedFile)) case errors.Is(err, mcp.ErrSignInFailed): diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index 95fadfed54b..dff27c19831 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -126,7 +126,7 @@ func TestMcpLogoutUnknownServiceIsAnError(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.gihtub"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.gihtub", mcpLogout, &out) require.Error(t, err) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) assert.Contains(t, err.Error(), "system.ai.gihtub is not an MCP service visible to profile logfood") assert.Empty(t, out.State) } @@ -174,7 +174,7 @@ func TestMcpLogoutEndpointNotFoundIsNotAServiceError(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.github"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogout, &out) require.Error(t, err) - assert.NotEqual(t, categoryServiceNotFound, errorCategory(err)) + assert.NotEqual(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) assert.NotContains(t, err.Error(), "is not an MCP service visible") } diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 5afdf3d86d2..14ad26c9db9 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -32,6 +32,9 @@ type removeOutput struct { Skipped []removeSkipped `json:"skipped"` Error string `json:"error,omitempty"` ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` + + // results is one telemetry result per (service, agent) attempted. It is not in the JSON. + results []protos.AitoolsMcpServerResult } type removeRemoved struct { @@ -97,12 +100,19 @@ removes one.`, databricks aitools mcp remove --profile logfood --agents cursor databricks aitools mcp remove --all`, Args: cobra.NoArgs, + // Resolve auth best-effort so telemetry can upload; see tryConfigureAuth. + PreRunE: tryConfigureAuth, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) out := &removeOutput{Removed: []removeRemoved{}, Skipped: []removeSkipped{}} runErr := runMcpRemove(ctx, cmd, f, jsonMode, out) + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationRemove, + Servers: out.results, + }) if !jsonMode { return runErr } @@ -237,10 +247,12 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // There is no config this CLI can edit for the agent, so only the record can go. removed.Agents = append(removed.Agents, agentName) removed.RecordOnly = append(removed.RecordOnly, agentName) + out.results = append(out.results, removeServerResult(e.Service, agentName, removeDeleted, nil)) drops = append(drops, recordDrop{name: e.Name, agent: agentName}) continue } outcome, reason, err := removeFromAgent(ctx, files, e, a, pin, len(agentNames) > 0) + out.results = append(out.results, removeServerResult(e.Service, agentName, outcome, err)) if err != nil { errs = append(errs, err) out.Skipped = append(out.Skipped, removeSkipped{Name: e.Name, Agent: agentName, Reason: fmt.Sprintf("could not update %s: %s", agentDisplayName(agentName), err)}) @@ -315,7 +327,7 @@ func selectInstalls(state *mcp.State, names []string, profileName string) ([]mcp return nil, err } if len(matched) == 0 { - return nil, &noRecordError{name: name, err: failure(categoryServiceNotFound, "no installed MCP server named %s%s", name, where)} + return nil, &noRecordError{name: name, err: failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "no installed MCP server named %s%s", name, where)} } for _, i := range matched { chosen[i] = true @@ -363,7 +375,7 @@ func withAdoptHint(ctx context.Context, err error, state *mcp.State) error { if u.fix != "" { hint += ". To adopt it, so that remove works, run `" + u.fix + "`" } - return withCategory(fmt.Errorf("%w; %s", err, hint), categoryServiceNotFound) + return withCategory(fmt.Errorf("%w; %s", err, hint), protos.AitoolsErrorCategoryMCPServiceNotFound) } return err } @@ -430,7 +442,7 @@ func checkTargetsInstalled(selected []mcp.Entry, targets []*agents.Agent) error _, ok := e.Agents[a.Name] return ok }) { - return failure(categoryServiceNotFound, "no selected MCP server is installed for %s", agentDisplayName(a.Name)) + return failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "no selected MCP server is installed for %s", agentDisplayName(a.Name)) } } return nil diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index bcffc4abfc0..19dbbe257bb 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -82,7 +82,7 @@ func TestSelectInstalls(t *testing.T) { _, err = selectInstalls(state, []string{"github-dogfood"}, "logfood") require.Error(t, err) assert.Equal(t, "no installed MCP server named github-dogfood on profile logfood", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) // A profile with nothing installed has nothing to remove, the same as --all on an empty state. none, err := selectInstalls(state, nil, "typo") @@ -468,7 +468,7 @@ func TestMcpRemoveNamesUnmanagedEntrySaysHowToAdopt(t *testing.T) { require.ErrorIs(t, err, root.ErrAlreadyPrinted) out := readRemoveOutput(t, stdout.String()) assert.Equal(t, "no installed MCP server named genie_one_mcp-logfood; Claude Code has an entry named genie_one_mcp-logfood that runs aitools mcp serve with no install record. To adopt it, so that remove works, run `databricks aitools mcp add --names system.ai.genie_one_mcp --profile logfood --agents claude-code`", out.Error) - assert.Equal(t, categoryServiceNotFound, out.ErrorCategory) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, out.ErrorCategory) } func TestMcpRemoveRefusesAgentWithNothingInstalled(t *testing.T) { @@ -482,7 +482,7 @@ func TestMcpRemoveRefusesAgentWithNothingInstalled(t *testing.T) { require.ErrorIs(t, err, root.ErrAlreadyPrinted) out := readRemoveOutput(t, stdout.String()) assert.Equal(t, "no selected MCP server is installed for Cursor", out.Error) - assert.Equal(t, categoryServiceNotFound, out.ErrorCategory) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, out.ErrorCategory) // Nothing was removed by the refusal. state, err := mcp.LoadState(t.Context()) @@ -897,7 +897,7 @@ func TestMcpRemovePickerRefusesAgentWithNoServer(t *testing.T) { err := runInteractive(t, "mcp", "remove", "--agents", "codex") require.Error(t, err) assert.Equal(t, "no selected MCP server is installed for "+agents.ByName(agents.NameCodex).DisplayName, err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) state, err := mcp.LoadState(t.Context()) require.NoError(t, err) assert.Len(t, state.Servers, 1) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index a14bcbe4461..8fd76219442 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -26,6 +26,7 @@ import ( "github.com/databricks/cli/libs/auth/storage" "github.com/databricks/cli/libs/auth/u2m" "github.com/databricks/cli/libs/browser" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/shellquote" @@ -220,6 +221,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { if err != nil { return serveFailure(cmd, err, serveBindStatus) } + agent := proxyAgentType(f.agent) cfg, entry, err := mcp.Bind(cmd.Context(), mcp.BindOptions{ Profile: profileName, Service: f.service, @@ -227,9 +229,14 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { Stderr: cmd.ErrOrStderr(), }) if err != nil { + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveBindFailure(cmd, profileName, err) } + // The bound profile authenticates the exit-time telemetry upload. + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), cfg)) + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, nil)) + // A server pinned to a config file names that file in its hints, as add and login do. pinnedFile := mcp.PinnedFile(entry, cfg.ConfigFile) ctx, stop := serveSignalContext(cmd.Context()) @@ -249,6 +256,9 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { } } challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile}, cmd.ErrOrStderr()) + if challenge != nil { + challenge.Recorder = mcpProxyTelemetry{agent: agent, service: f.service} + } if err != nil { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go new file mode 100644 index 00000000000..ad05d2f6646 --- /dev/null +++ b/cmd/aitools/mcp_telemetry.go @@ -0,0 +1,228 @@ +package aitools + +import ( + "context" + "strings" + + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" +) + +// mcpServiceTypes maps the system.ai leaves in the CLI's service table to their telemetry enum. +var mcpServiceTypes = map[string]protos.AitoolsMcpService{ + "genie_one_mcp": protos.AitoolsMcpServiceGenieOne, + "dbsql": protos.AitoolsMcpServiceDBSQL, + "web_search": protos.AitoolsMcpServiceWebSearch, + "sandbox": protos.AitoolsMcpServiceSandbox, + "slack": protos.AitoolsMcpServiceSlack, + "github": protos.AitoolsMcpServiceGitHub, + "atlassian": protos.AitoolsMcpServiceAtlassian, + "google_drive": protos.AitoolsMcpServiceGoogleDrive, + "google_calendar": protos.AitoolsMcpServiceGoogleCalendar, + "gmail": protos.AitoolsMcpServiceGmail, + "microsoft_365": protos.AitoolsMcpServiceMicrosoft365, +} + +// mcpServiceType maps a service full name to its telemetry enum. The name itself is never sent: +// a system.ai leaf in the table has its own value, other system.ai leaves are OTHER_SYSTEM, and +// services outside system.ai are CUSTOM. +func mcpServiceType(fullName string) protos.AitoolsMcpService { + parts := strings.Split(fullName, ".") + if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], "system.ai") { + return protos.AitoolsMcpServiceCustom + } + if t, ok := mcpServiceTypes[strings.ToLower(parts[2])]; ok { + return t + } + return protos.AitoolsMcpServiceOtherSystem +} + +// callingAgentType returns the agent that drives the CLI, Unspecified when none is detected. +func callingAgentType() protos.AitoolsAgentType { + a := agents.CallingAgent() + if a == nil { + return protos.AitoolsAgentTypeUnspecified + } + return agentType(a.Name) +} + +// proxyAgentType returns the agent that started the proxy: the --agent flag when given, else the +// detected agent. +func proxyAgentType(flag string) protos.AitoolsAgentType { + if flag == "" { + return callingAgentType() + } + return agentType(flag) +} + +// logMcpEvent buffers an MCP event; cmd/root uploads it at exit. An event without server results +// is not logged: a run that stopped before any server was reached has nothing to report. The bound +// profile count is read from the state file as it is after the operation. A state file that cannot +// be read drops the event; telemetry never fails a command. +func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) { + if len(event.Servers) == 0 { + return + } + state, err := mcp.LoadState(ctx) + if err != nil { + log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) + return + } + event.BoundProfileCount = boundProfileCount(state) + telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsMcpEvent: &event}) +} + +// boundProfileCount returns the number of distinct profiles that have servers installed. +func boundProfileCount(state *mcp.State) int64 { + profiles := map[string]bool{} + for _, e := range state.Servers { + profiles[e.Profile] = true + } + return int64(len(profiles)) +} + +// addSignInState is the sign-in state of a server after an add: the state that --login reports, else PENDING for a +// server that needs sign-in, else NOT_REQUIRED. +func addSignInState(s addServerOutput) protos.AitoolsMcpSignInState { + if s.SignIn != nil { + return signInStateOf(s.SignIn.State) + } + if s.SignInRequired { + return protos.AitoolsMcpSignInStatePending + } + return protos.AitoolsMcpSignInStateNotRequired +} + +// addServerResults returns one result per (service, agent) of an add run. Installed entries are +// CHANGED, re-asserted ones UNCHANGED, and failed ones FAILED with their category. +func addServerResults(servers []addServerOutput) []protos.AitoolsMcpServerResult { + var results []protos.AitoolsMcpServerResult + for _, s := range servers { + for _, a := range s.Agents { + r := protos.AitoolsMcpServerResult{ + Service: mcpServiceType(s.Service), + Agent: agentType(a.Name), + SignInState: addSignInState(s), + } + switch a.Status { + case statusInstalled: + r.Outcome = protos.AitoolsTargetOutcomeChanged + case statusReasserted: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + default: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = a.ErrorCategory + } + results = append(results, r) + } + } + return results +} + +// removeServerResult returns the result of removing one server from one agent config. An entry that +// is gone already, or that was changed outside this CLI, is SKIPPED: nothing was changed there. +func removeServerResult(service, agent string, outcome removeOutcome, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), Agent: agentType(agent)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case outcome == removeDeleted: + r.Outcome = protos.AitoolsTargetOutcomeChanged + default: + r.Outcome = protos.AitoolsTargetOutcomeSkipped + } + return r +} + +// signInStateOf maps a sign-in state name of login to its telemetry state. Anything but ACTIVE or +// NOT_REQUIRED is still PENDING. +func signInStateOf(state string) protos.AitoolsMcpSignInState { + switch state { + case signInStateActive: + return protos.AitoolsMcpSignInStateActive + case signInStateNotRequired: + return protos.AitoolsMcpSignInStateNotRequired + } + return protos.AitoolsMcpSignInStatePending +} + +// signInServerResult returns the result of one provider sign-in. The sign-in was already active +// (alreadySignedIn) or the service needed none, so nothing changed: UNCHANGED. +func signInServerResult(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), SignInState: signInStateOf(state)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case alreadySignedIn || state == signInStateNotRequired: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + default: + r.Outcome = protos.AitoolsTargetOutcomeChanged + } + return r +} + +// signOutServerResult returns the result of a logout. Logout does not read whether the service needs +// sign-in, so the sign-in state is not set. +func signOutServerResult(service, state string, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case state == signOutStateDeleted: + r.Outcome = protos.AitoolsTargetOutcomeChanged + default: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + } + return r +} + +// proxyStartedEvent is the PROXY_STARTED event of serve. err is the binding failure, or nil when the +// proxy is about to serve. The event only uploads when the bound profile resolved, so a failed +// binding usually leaves it unsent. +func proxyStartedEvent(agent protos.AitoolsAgentType, service string, err error) protos.AitoolsMcpEvent { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} + if err != nil { + r.ErrorCategory = errorCategory(err) + } + return protos.AitoolsMcpEvent{ + CallingAgent: agent, + Operation: protos.AitoolsMcpOperationProxyStarted, + Servers: []protos.AitoolsMcpServerResult{r}, + } +} + +// mcpProxyTelemetry is the proxy's telemetry for one serve session. It implements proxy.Recorder. +type mcpProxyTelemetry struct { + agent protos.AitoolsAgentType + service string +} + +// SignInPrompted logs SIGN_IN_PROMPTED once the sign-in page the authenticate tool opened has been +// waited on. signedIn reports whether the credential became ACTIVE. +func (t mcpProxyTelemetry) SignInPrompted(ctx context.Context, signedIn bool) { + logMcpEvent(ctx, signInPromptedEvent(t.agent, t.service, signedIn)) +} + +// signInPromptedEvent is the SIGN_IN_PROMPTED event of serve: the sign-in is required, and the +// sign-in state is ACTIVE when the wait ended with the credential, PENDING otherwise. +func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn bool) protos.AitoolsMcpEvent { + state := protos.AitoolsMcpSignInStatePending + if signedIn { + state = protos.AitoolsMcpSignInStateActive + } + return protos.AitoolsMcpEvent{ + CallingAgent: agent, + Operation: protos.AitoolsMcpOperationSignInPrompted, + Servers: []protos.AitoolsMcpServerResult{{ + Service: mcpServiceType(service), + ErrorCategory: protos.AitoolsErrorCategorySignInRequired, + SignInState: state, + }}, + } +} diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go new file mode 100644 index 00000000000..11cb852b8ef --- /dev/null +++ b/cmd/aitools/mcp_telemetry_test.go @@ -0,0 +1,235 @@ +package aitools + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMcpServiceType(t *testing.T) { + // Every leaf in the CLI's known table has its own enum, so a known service never reports OTHER_SYSTEM. + cases := map[string]protos.AitoolsMcpService{ + "system.ai.genie_one_mcp": protos.AitoolsMcpServiceGenieOne, + "system.ai.dbsql": protos.AitoolsMcpServiceDBSQL, + "system.ai.web_search": protos.AitoolsMcpServiceWebSearch, + "system.ai.sandbox": protos.AitoolsMcpServiceSandbox, + "system.ai.slack": protos.AitoolsMcpServiceSlack, + "system.ai.github": protos.AitoolsMcpServiceGitHub, + "system.ai.atlassian": protos.AitoolsMcpServiceAtlassian, + "system.ai.google_drive": protos.AitoolsMcpServiceGoogleDrive, + "system.ai.google_calendar": protos.AitoolsMcpServiceGoogleCalendar, + "system.ai.gmail": protos.AitoolsMcpServiceGmail, + "system.ai.microsoft_365": protos.AitoolsMcpServiceMicrosoft365, + // The schema name is case-insensitive in Unity Catalog. + "System.AI.GitHub": protos.AitoolsMcpServiceGitHub, + // A system.ai leaf the CLI does not know yet. + "system.ai.future_tool": protos.AitoolsMcpServiceOtherSystem, + // Anything outside system.ai, whatever its name. + "main.tools.github": protos.AitoolsMcpServiceCustom, + "system.ai": protos.AitoolsMcpServiceCustom, + "system.ai.a.github": protos.AitoolsMcpServiceCustom, + } + for name, want := range cases { + assert.Equal(t, want, mcpServiceType(name), name) + } +} + +func TestAddServerResults(t *testing.T) { + servers := []addServerOutput{ + { + Service: "system.ai.github", + SignInRequired: true, + SignIn: &addSignInOutput{State: signInStateActive}, + Agents: []addAgentOutput{ + {Name: "claude-code", Status: statusInstalled}, + {Name: "codex", Status: statusReasserted}, + {Name: "cursor", Status: statusFailed, ErrorCategory: protos.AitoolsErrorCategoryAgentConfigInvalid}, + }, + }, + { + Service: "main.tools.custom", + Agents: []addAgentOutput{{Name: "claude-code", Status: statusInstalled}}, + }, + } + + assert.Equal(t, []protos.AitoolsMcpServerResult{ + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeClaudeCode, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCodex, + Outcome: protos.AitoolsTargetOutcomeUnchanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCursor, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryAgentConfigInvalid, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceCustom, + Agent: protos.AitoolsAgentTypeClaudeCode, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStateNotRequired, + }, + }, addServerResults(servers)) +} + +func TestRemoveServerResult(t *testing.T) { + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, + removeServerResult("system.ai.github", "claude-code", removeDeleted, nil).Outcome) + // An entry already gone, or changed outside this CLI, is left as it is. + assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, + removeServerResult("system.ai.github", "claude-code", removePruned, nil).Outcome) + assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, + removeServerResult("system.ai.github", "claude-code", removeKept, nil).Outcome) + + failed := removeServerResult("system.ai.github", "codex", removeDeleted, errors.New("config is not JSON")) + assert.Equal(t, protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCodex, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryUncategorized, + }, failed) +} + +func TestSignInServerResult(t *testing.T) { + cases := []struct { + name string + state string + alreadySignedIn bool + err error + want protos.AitoolsMcpServerResult + }{ + { + name: "signed in now", + state: signInStateActive, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeChanged, SignInState: protos.AitoolsMcpSignInStateActive}, + }, + { + name: "already active", + state: signInStateActive, + alreadySignedIn: true, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateActive}, + }, + { + name: "no sign-in needed", + state: signInStateNotRequired, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateNotRequired}, + }, + { + name: "timed out", + state: signInStateTimeout, + err: failure(protos.AitoolsErrorCategorySignInTimeout, nil, "timed out"), + want: protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategorySignInTimeout, SignInState: protos.AitoolsMcpSignInStatePending, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, signInServerResult("system.ai.github", tc.state, tc.alreadySignedIn, tc.err)) + }) + } +} + +func TestSignOutServerResult(t *testing.T) { + // Logout does not read the sign-in state, so the result has none. + assert.Equal(t, protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeChanged}, + signOutServerResult("system.ai.github", signOutStateDeleted, nil)) + assert.Equal(t, protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged}, + signOutServerResult("system.ai.github", signOutStateNotSignedIn, nil)) +} + +func TestProxyStartedEvent(t *testing.T) { + ok := proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp", nil) + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeClaudeCode, + Operation: protos.AitoolsMcpOperationProxyStarted, + Servers: []protos.AitoolsMcpServerResult{{Service: protos.AitoolsMcpServiceGenieOne}}, + }, ok) + + // Each binding failure has its category; the error text never reaches the event. + for category, bind := range map[protos.AitoolsErrorCategory]mcp.BindCategory{ + protos.AitoolsErrorCategoryProfileNotFound: mcp.BindProfileNotFound, + protos.AitoolsErrorCategoryProfileHostChanged: mcp.BindProfileHostChanged, + protos.AitoolsErrorCategoryAuthFailed: mcp.BindAuthFailed, + } { + event := proxyStartedEvent(protos.AitoolsAgentTypeUnspecified, "main.tools.custom", &mcp.BindError{Category: bind}) + require.Len(t, event.Servers, 1) + assert.Equal(t, category, event.Servers[0].ErrorCategory) + assert.Equal(t, protos.AitoolsMcpServiceCustom, event.Servers[0].Service) + } +} + +func TestSignInPromptedEvent(t *testing.T) { + for signedIn, state := range map[bool]protos.AitoolsMcpSignInState{ + true: protos.AitoolsMcpSignInStateActive, + false: protos.AitoolsMcpSignInStatePending, + } { + event := signInPromptedEvent(protos.AitoolsAgentTypeCursor, "system.ai.github", signedIn) + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeCursor, + Operation: protos.AitoolsMcpOperationSignInPrompted, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + ErrorCategory: protos.AitoolsErrorCategorySignInRequired, + SignInState: state, + }}, + }, event) + } +} + +func TestBoundProfileCount(t *testing.T) { + state := &mcp.State{Servers: []mcp.Entry{ + {Profile: "logfood", Service: "system.ai.github"}, + {Profile: "logfood", Service: "system.ai.genie_one_mcp"}, + {Profile: "dogfood", Service: "system.ai.github"}, + }} + assert.Equal(t, int64(2), boundProfileCount(state)) + assert.Equal(t, int64(0), boundProfileCount(&mcp.State{})) +} + +// The event carries enums and counts only. A custom service and the host of its profile must not +// appear anywhere in the payload. +func TestMcpEventOmitsUserContent(t *testing.T) { + event := protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeClaudeCode, + Operation: protos.AitoolsMcpOperationAdd, + BoundProfileCount: 1, + Servers: addServerResults([]addServerOutput{{ + Service: "acme_catalog.secret_schema.internal_tool", + Agents: []addAgentOutput{{Name: "claude-code", Status: statusInstalled}}, + }}), + } + b, err := json.Marshal(event) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(b, &got)) + assert.ElementsMatch(t, []string{"calling_agent", "operation", "servers", "bound_profile_count"}, keys(got)) + assert.NotContains(t, string(b), "acme_catalog") + assert.NotContains(t, string(b), "secret_schema") + assert.NotContains(t, string(b), "internal_tool") + assert.Contains(t, string(b), `"service":"CUSTOM"`) +} + +func keys(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/cmd/aitools/mcp_test.go b/cmd/aitools/mcp_test.go index 3f9635798f7..011e718b94c 100644 --- a/cmd/aitools/mcp_test.go +++ b/cmd/aitools/mcp_test.go @@ -108,7 +108,7 @@ host = https://b.test _, err := resolveProfile(ctx, "") require.Error(t, err) assert.Equal(t, "several workspace profiles exist; pass --profile. Found: a, b", err.Error()) - assert.Equal(t, categoryProfileRequired, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryProfileRequired, errorCategory(err)) }) t.Run("no workspace profiles is an error naming the file", func(t *testing.T) { @@ -116,7 +116,7 @@ host = https://b.test _, err := resolveProfile(ctx, "") require.Error(t, err) assert.Contains(t, err.Error(), "no workspace profiles in ") - assert.Equal(t, categoryNoProfile, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryNoProfile, errorCategory(err)) assert.Contains(t, err.Error(), "databricks auth login --host --profile ") }) } @@ -126,11 +126,11 @@ func TestErrorCategoryUsesTypedErrorsOnly(t *testing.T) { assert.Equal(t, protos.AitoolsErrorCategoryUnknownAgent, errorCategory(typed)) assert.Equal(t, "unknown agent \"pi\"", typed.Error()) - wrapped := withCategory(fmt.Errorf("%s: %w", "boom", errors.New("cause")), categoryServiceNotFound) - assert.Equal(t, categoryServiceNotFound, errorCategory(wrapped)) + wrapped := withCategory(fmt.Errorf("%s: %w", "boom", errors.New("cause")), protos.AitoolsErrorCategoryMCPServiceNotFound) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(wrapped)) configInvalid := &os.PathError{Op: "parse", Path: "x", Err: agentconfig.ErrConfigInvalid} - assert.Equal(t, categoryAgentConfigInvalid, errorCategory(configInvalid)) + assert.Equal(t, protos.AitoolsErrorCategoryAgentConfigInvalid, errorCategory(configInvalid)) assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, errorCategory(errors.New("some failure"))) } diff --git a/libs/aitools/mcp/proxy/bridge_test.go b/libs/aitools/mcp/proxy/bridge_test.go index 5dd0e840f9e..bee91aa7419 100644 --- a/libs/aitools/mcp/proxy/bridge_test.go +++ b/libs/aitools/mcp/proxy/bridge_test.go @@ -2375,7 +2375,7 @@ func TestSignInWritesShareTheBridgeStderr(t *testing.T) { Config: &config.Config{Host: testHost}, Authenticate: func(*http.Request) error { return nil }, Profile: "logfood", - Challenge: &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn}, + Challenge: &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn, Recorder: &promptRecorder{}}, }) }() diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index f1239cf8d90..e7f2ffc1e5a 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -50,6 +50,14 @@ type SignInRequest struct { // cancelled. Its error is the answer's text. type SignInFunc func(ctx context.Context, req SignInRequest) (alreadySignedIn, urlShown bool, err error) +// Recorder receives the proxy's telemetry events. The command implements it, so the proxy does not +// build telemetry events itself. +type Recorder interface { + // SignInPrompted runs after the wait of an authenticate call that opened the sign-in page. + // signedIn reports whether the credential became ACTIVE. + SignInPrompted(ctx context.Context, signedIn bool) +} + // Challenge turns on sign-in handling for a service that needs a provider sign-in. type Challenge struct { // Service is the full name of the MCP service, for example system.ai.github. @@ -73,6 +81,8 @@ type Challenge struct { // SignIn runs the sign-in. The command provides it, so the proxy never polls or opens a browser itself. Its error is // the answer's text, so it names the service, the profile and the fix. SignIn SignInFunc + // Recorder logs the sign-in prompt. The command provides it. + Recorder Recorder } // signInHooks applies the sign-in rules to the bridge. Services without a challenge get no hooks. @@ -499,13 +509,18 @@ func (h *signInHooks) authenticate(ctx context.Context, id, progressToken json.R return nil } + open := !h.pageCancelled alreadySignedIn, urlShown, err := h.c.SignIn(ctx, SignInRequest{ - Open: !h.pageCancelled, + Open: open, Progress: func(pageURL string) { p.report(ctx, pageURL) }, Stderr: h.stderr, }) + // A caller that was already signed in opened no page, so there is no prompt to record. + if open && !alreadySignedIn { + h.c.Recorder.SignInPrompted(ctx, err == nil) + } h.pageCancelled = urlShown && err != nil && ctx.Err() != nil h.stateMu.Lock() h.alreadySignedIn = err == nil && alreadySignedIn diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index d674c380fd4..57a9ecf3e72 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -68,6 +68,24 @@ func (s *signInStub) requests() []SignInRequest { // loginCommand is the hint that the command builds for a pinned config file and a profile name with a space. const loginCommand = `DATABRICKS_CONFIG_FILE='/work/cfg/my file.cfg' databricks aitools mcp login system.ai.github --profile 'my profile'` +// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. +type promptRecorder struct { + mu sync.Mutex + signedIn []bool +} + +func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { + r.mu.Lock() + defer r.mu.Unlock() + r.signedIn = append(r.signedIn, signedIn) +} + +func (r *promptRecorder) prompts() []bool { + r.mu.Lock() + defer r.mu.Unlock() + return append([]bool(nil), r.signedIn...) +} + // challengedBridge returns a bridge to url with the sign-in hooks of a service that needs sign-in. func challengedBridge(url string, signIn SignInFunc) *Bridge { b := bridgeFor(url) @@ -255,6 +273,37 @@ func TestAuthenticateSaysAlreadySignedInAfterSignIn(t *testing.T) { require.NoError(t, p.finish(t)) } +// The prompt is recorded once per page that was opened, after its wait, with whether the credential +// became ACTIVE. A call that opens no page, or finds the caller already signed in, records nothing. +func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { + srv, _ := challengedService(t) + signIn := &signInStub{reply: func(call int) (bool, error) { + if call == 1 { + return false, errors.New("sign-in did not complete") + } + return call == 3, nil + }} + recorder := &promptRecorder{} + c := testChallenge(signIn.SignIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + authenticate := `{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}` + challenge(t, p) + + p.send(t, fmt.Sprintf(authenticate, 1)) + p.waitLines(t, 2) + p.send(t, fmt.Sprintf(authenticate, 2)) + p.waitLines(t, 4) + p.send(t, fmt.Sprintf(authenticate, 3)) + p.waitLines(t, 6) + + assert.Equal(t, []bool{true, true, true}, signIn.calls(), "every call opens the page") + assert.Equal(t, []bool{false, true}, recorder.prompts(), "call 3 was already signed in and records nothing") + require.NoError(t, p.finish(t)) +} + func TestAuthenticateReopensPageAfterTimeout(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { @@ -343,7 +392,7 @@ func TestChallengeHintsPrintTheLoginCommand(t *testing.T) { return false, true, errors.New("sign-in to system.ai.github did not complete in 5 minutes. Finish at " + testURL + " then retry: " + loginCommand) } b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "my profile", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "my profile", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn, Recorder: &promptRecorder{}}, io.Discard) p := startPiped(t, t.Context(), b) p.send(t, `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"echo","arguments":{}}}`) @@ -1001,6 +1050,7 @@ func testChallenge(signIn SignInFunc) *Challenge { LoginCommand: "databricks aitools mcp login system.ai.github --profile logfood", LogoutCommand: "databricks aitools mcp logout system.ai.github --profile logfood", SignIn: signIn, + Recorder: &promptRecorder{}, } } diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index 406dde89a58..ead0a7b76c8 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -74,6 +74,23 @@ const ( // The command's context was cancelled. SIGINT outside a prompt is not // counted here: it ends the process before telemetry is uploaded. AitoolsErrorCategoryCommandInterrupted AitoolsErrorCategory = "COMMAND_INTERRUPTED" + + // `aitools mcp`: profiles and sign-in. + AitoolsErrorCategoryNoProfile AitoolsErrorCategory = "NO_PROFILE" + AitoolsErrorCategoryProfileRequired AitoolsErrorCategory = "PROFILE_REQUIRED" + AitoolsErrorCategoryProfileNotFound AitoolsErrorCategory = "PROFILE_NOT_FOUND" + AitoolsErrorCategoryProfileHostChanged AitoolsErrorCategory = "PROFILE_HOST_CHANGED" + AitoolsErrorCategoryAuthFailed AitoolsErrorCategory = "AUTH_FAILED" + AitoolsErrorCategorySignInRequired AitoolsErrorCategory = "SIGN_IN_REQUIRED" + AitoolsErrorCategorySignInTimeout AitoolsErrorCategory = "SIGN_IN_TIMEOUT" + + // `aitools mcp`: the service list, agent configs and the proxy. + AitoolsErrorCategoryMCPServiceNotFound AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" + AitoolsErrorCategoryMCPPermissionDenied AitoolsErrorCategory = "MCP_PERMISSION_DENIED" + AitoolsErrorCategoryMCPLocationNotFound AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" + AitoolsErrorCategoryMCPNameConflict AitoolsErrorCategory = "MCP_NAME_CONFLICT" + AitoolsErrorCategoryMCPUpstreamError AitoolsErrorCategory = "MCP_UPSTREAM_ERROR" + AitoolsErrorCategoryAgentConfigInvalid AitoolsErrorCategory = "AGENT_CONFIG_INVALID" ) // AitoolsAgentResult records one agent's failed or skipped outcome within an diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go new file mode 100644 index 00000000000..c42dbc20285 --- /dev/null +++ b/libs/telemetry/protos/aitools_mcp.go @@ -0,0 +1,94 @@ +package protos + +// The types in this file mirror the aitools MCP messages of the databricks_cli +// lumberjack protos: AitoolsMcpEvent and its server result in +// proto/logs/frontend/databricks_cli/aitools_mcp.proto, and the enums in +// proto/logs/frontend/databricks_cli/enum.proto. + +// AitoolsMcpOperation mirrors AitoolsMcpOperation.Type in aitools_mcp.proto. +type AitoolsMcpOperation string + +const ( + AitoolsMcpOperationAdd AitoolsMcpOperation = "ADD" + AitoolsMcpOperationRemove AitoolsMcpOperation = "REMOVE" + AitoolsMcpOperationSignIn AitoolsMcpOperation = "SIGN_IN" + AitoolsMcpOperationSignOut AitoolsMcpOperation = "SIGN_OUT" + AitoolsMcpOperationProxyStarted AitoolsMcpOperation = "PROXY_STARTED" + AitoolsMcpOperationSignInPrompted AitoolsMcpOperation = "SIGN_IN_PROMPTED" +) + +// AitoolsMcpService mirrors AitoolsMcpService.Type in enum.proto. The CLI +// sends only these values, never a service name: the system.ai leaves in the +// table map to their own value, other system.ai leaves to OtherSystem, and any +// service outside system.ai to Custom. +type AitoolsMcpService string + +const ( + AitoolsMcpServiceOtherSystem AitoolsMcpService = "OTHER_SYSTEM" + AitoolsMcpServiceCustom AitoolsMcpService = "CUSTOM" + AitoolsMcpServiceGenieOne AitoolsMcpService = "GENIE_ONE" + AitoolsMcpServiceDBSQL AitoolsMcpService = "DBSQL" + AitoolsMcpServiceWebSearch AitoolsMcpService = "WEB_SEARCH" + AitoolsMcpServiceSandbox AitoolsMcpService = "SANDBOX" + AitoolsMcpServiceSlack AitoolsMcpService = "SLACK" + AitoolsMcpServiceGitHub AitoolsMcpService = "GITHUB" + AitoolsMcpServiceAtlassian AitoolsMcpService = "ATLASSIAN" + AitoolsMcpServiceGoogleDrive AitoolsMcpService = "GOOGLE_DRIVE" + AitoolsMcpServiceGoogleCalendar AitoolsMcpService = "GOOGLE_CALENDAR" + AitoolsMcpServiceGmail AitoolsMcpService = "GMAIL" + AitoolsMcpServiceMicrosoft365 AitoolsMcpService = "MICROSOFT_365" +) + +// AitoolsMcpSignInState mirrors AitoolsMcpSignInState.Type in enum.proto. It is +// the sign-in state of a service after the operation. +type AitoolsMcpSignInState string + +const ( + AitoolsMcpSignInStateNotRequired AitoolsMcpSignInState = "NOT_REQUIRED" + AitoolsMcpSignInStateActive AitoolsMcpSignInState = "ACTIVE" + AitoolsMcpSignInStatePending AitoolsMcpSignInState = "PENDING" +) + +// AitoolsTargetOutcome mirrors AitoolsTargetOutcome.Type in enum.proto. It is +// what an operation did to one target: an agent config entry, or a sign-in. +type AitoolsTargetOutcome string + +const ( + AitoolsTargetOutcomeChanged AitoolsTargetOutcome = "CHANGED" + AitoolsTargetOutcomeUnchanged AitoolsTargetOutcome = "UNCHANGED" + AitoolsTargetOutcomeUpdateAvailable AitoolsTargetOutcome = "UPDATE_AVAILABLE" + AitoolsTargetOutcomeSkipped AitoolsTargetOutcome = "SKIPPED" + AitoolsTargetOutcomeFailed AitoolsTargetOutcome = "FAILED" +) + +// AitoolsMcpServerResult is the result for one service, for one agent when the +// operation writes agent config. Unset fields are omitted: an agent is not set +// for sign-in, an outcome is not set for proxy events, and an error category is +// set only when the target failed or was skipped. +type AitoolsMcpServerResult struct { + Service AitoolsMcpService `json:"service"` + Agent AitoolsAgentType `json:"agent,omitempty"` + Outcome AitoolsTargetOutcome `json:"outcome,omitempty"` + ErrorCategory AitoolsErrorCategory `json:"error_category,omitempty"` + SignInState AitoolsMcpSignInState `json:"sign_in_state,omitempty"` +} + +// AitoolsMcpEvent is emitted by `databricks aitools mcp` commands and by the +// proxy (`serve`). It carries enums and counts only: no host, profile, workspace +// ID, custom service or schema name, path, error text or tool name. +type AitoolsMcpEvent struct { + // CallingAgent is the coding agent that ran the command or started the proxy. + // AitoolsAgentTypeUnspecified when no agent is detected. + CallingAgent AitoolsAgentType `json:"calling_agent"` + + Operation AitoolsMcpOperation `json:"operation"` + + // Servers holds one entry per (service, agent) for ADD and REMOVE, one per + // service for SIGN_IN, SIGN_OUT, PROXY_STARTED and SIGN_IN_PROMPTED. + Servers []AitoolsMcpServerResult `json:"servers,omitempty"` + + // BoundProfileCount is the number of distinct profiles with MCP servers + // installed, read from the state file after the operation. Zero is a real + // value (every server removed), so it is not omitted. + BoundProfileCount int64 `json:"bound_profile_count"` +} diff --git a/libs/telemetry/protos/aitools_mcp_test.go b/libs/telemetry/protos/aitools_mcp_test.go new file mode 100644 index 00000000000..3b6ef512e3a --- /dev/null +++ b/libs/telemetry/protos/aitools_mcp_test.go @@ -0,0 +1,57 @@ +package protos + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The lumberjack proto keys these fields in snake_case. This pins the wire keys of an event +// and of one server result, and that unset enums are omitted rather than sent empty. +func TestAitoolsMcpEventEncodesSnakeCaseKeys(t *testing.T) { + b, err := json.Marshal(DatabricksCliLog{AitoolsMcpEvent: &AitoolsMcpEvent{ + CallingAgent: AitoolsAgentTypeClaudeCode, + Operation: AitoolsMcpOperationAdd, + BoundProfileCount: 0, + Servers: []AitoolsMcpServerResult{ + { + Service: AitoolsMcpServiceGitHub, + Agent: AitoolsAgentTypeCodex, + Outcome: AitoolsTargetOutcomeFailed, + ErrorCategory: AitoolsErrorCategoryAgentConfigInvalid, + SignInState: AitoolsMcpSignInStatePending, + }, + }, + }}) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(b, &got)) + event, ok := got["aitools_mcp_event"].(map[string]any) + require.True(t, ok, "aitools_mcp_event must be the event key") + + assert.Equal(t, "CLAUDE_CODE", event["calling_agent"]) + assert.Equal(t, "ADD", event["operation"]) + // Zero is a real count and is sent. + assert.EqualValues(t, 0, event["bound_profile_count"]) + + servers := event["servers"].([]any) + require.Len(t, servers, 1) + server := servers[0].(map[string]any) + assert.Equal(t, map[string]any{ + "service": "GITHUB", + "agent": "CODEX", + "outcome": "FAILED", + "error_category": "AGENT_CONFIG_INVALID", + "sign_in_state": "PENDING", + }, server) +} + +// A proxy event has no agent on the result, no outcome and no error category when it succeeded. +func TestAitoolsMcpServerResultOmitsUnsetFields(t *testing.T) { + b, err := json.Marshal(AitoolsMcpServerResult{Service: AitoolsMcpServiceOtherSystem}) + require.NoError(t, err) + assert.JSONEq(t, `{"service":"OTHER_SYSTEM"}`, string(b)) +} diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index f3c37736cc1..dcaf17d8e92 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -24,6 +24,7 @@ type DatabricksCliLog struct { SshAgentShimEvent *SshAgentShimEvent `json:"ssh_agent_shim_event,omitempty"` BundleConfigRemoteSyncEvent *BundleConfigRemoteSyncEvent `json:"bundle_config_remote_sync_event,omitempty"` AitoolsInstallEvent *AitoolsInstallEvent `json:"aitools_install_event,omitempty"` + AitoolsMcpEvent *AitoolsMcpEvent `json:"aitools_mcp_event,omitempty"` SetupLocalEvent *SetupLocalEvent `json:"setup_local_event,omitempty"` AirRunEvent *AirRunEvent `json:"air_run_event,omitempty"` AitoolsHintEvent *AitoolsHintEvent `json:"aitools_hint_event,omitempty"` From dd69f362207a9bc434ca48a1ef26273842af2ef1 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Thu, 12 Nov 2026 14:46:41 +0000 Subject: [PATCH 02/43] Create the agent config dirs in the telemetry acceptance cases mcp add now refuses an agent named with --agents whose config directory is missing. These cases add for claude-code (and codex in mcp-add) in a fresh HOME, so they create the directories first. The telemetry events do not change. Co-authored-by: Isaac --- acceptance/cmd/aitools/telemetry/mcp-add/script | 1 + acceptance/cmd/aitools/telemetry/mcp-remove/script | 1 + acceptance/cmd/aitools/telemetry/mcp-sign-in/script | 1 + 3 files changed, 3 insertions(+) diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/script b/acceptance/cmd/aitools/telemetry/mcp-add/script index 83d6b223112..2280d098810 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-add/script +++ b/acceptance/cmd/aitools/telemetry/mcp-add/script @@ -1,4 +1,5 @@ sethome home +mkdir -p home/.claude home/.codex envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index b25e4b4a31a..3d2481d9f43 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -1,4 +1,5 @@ sethome home +mkdir -p home/.claude envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script index 943af911b86..72025e3cde6 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script @@ -1,4 +1,5 @@ sethome home +mkdir -p home/.claude envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg From be6a80c530c9a1bdfb4a0467727cd6b356b4dfb5 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:46:54 +0000 Subject: [PATCH 03/43] Record the sign-in prompt only when a page URL was shown A sign-in that fails before its URL is printed, such as a read of the workspace ID, prompted nobody, so it no longer logs SIGN_IN_PROMPTED. The serve command sets the proxy's recorder where it builds the challenge, so the challenge is never handed over without one. Co-authored-by: Isaac --- cmd/aitools/mcp_serve.go | 12 ++++++------ libs/aitools/mcp/proxy/challenge.go | 7 ++++--- libs/aitools/mcp/proxy/challenge_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 8fd76219442..005805e8def 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -30,6 +30,7 @@ import ( "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/config/credentials" @@ -255,10 +256,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return serveFailure(cmd, err, 1) } } - challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile}, cmd.ErrOrStderr()) - if challenge != nil { - challenge.Recorder = mcpProxyTelemetry{agent: agent, service: f.service} - } + challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile, agent: agent}, cmd.ErrOrStderr()) if err != nil { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. @@ -350,14 +348,15 @@ func (g guardCredentials) Configure(context.Context, *config.Config) (credential return credentials.CredentialsProviderFn(g.refresh.authenticateCall), nil } -// serveTarget is the server that a serve session bridges: its service, the bound profile, the install record, and the -// config file it is pinned to. w is the profile's client, which must be guardedClient's. +// serveTarget is the server that a serve session bridges: its service, the bound profile, the install record, the config +// file it is pinned to, and the agent that started it. w is the profile's client, which must be guardedClient's. type serveTarget struct { w *databricks.WorkspaceClient entry *mcp.Entry profile string service string pinnedFile string + agent protos.AitoolsAgentType } // serveChallenge returns the sign-in handling of a service that needs provider sign-in, or nil for @@ -400,6 +399,7 @@ func serveChallenge(ctx context.Context, t serveTarget, errOut io.Writer) (*prox LogoutCommand: mcpLogoutHint(t.service, t.profile, t.pinnedFile), PinnedFile: t.pinnedFile, SignIn: serveSignIn(t), + Recorder: mcpProxyTelemetry{agent: t.agent, service: t.service}, }, nil } diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index e7f2ffc1e5a..1dbcffb0391 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -81,7 +81,7 @@ type Challenge struct { // SignIn runs the sign-in. The command provides it, so the proxy never polls or opens a browser itself. Its error is // the answer's text, so it names the service, the profile and the fix. SignIn SignInFunc - // Recorder logs the sign-in prompt. The command provides it. + // Recorder logs the sign-in prompt. The command provides it; it must not be nil. Recorder Recorder } @@ -517,8 +517,9 @@ func (h *signInHooks) authenticate(ctx context.Context, id, progressToken json.R }, Stderr: h.stderr, }) - // A caller that was already signed in opened no page, so there is no prompt to record. - if open && !alreadySignedIn { + // A prompt is a page the user was shown. A caller that was already signed in, or a sign-in that failed before its + // URL was printed, prompted nobody. + if open && !alreadySignedIn && urlShown { h.c.Recorder.SignInPrompted(ctx, err == nil) } h.pageCancelled = urlShown && err != nil && ctx.Err() != nil diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 57a9ecf3e72..9abbc23ff59 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -304,6 +304,30 @@ func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { require.NoError(t, p.finish(t)) } +// A sign-in that fails before its URL is printed, such as a read of the workspace ID, shows no page: nothing is recorded. +func TestAuthenticateRecordsNoPromptWhenNoURLShown(t *testing.T) { + srv, _ := challengedService(t) + var calls int + signIn := func(context.Context, SignInRequest) (bool, bool, error) { + calls++ + return false, false, errors.New("cannot read the workspace ID") + } + recorder := &promptRecorder{} + c := testChallenge(signIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + challenge(t, p) + + p.send(t, `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}`) + p.waitLines(t, 2) + + assert.Equal(t, 1, calls, "the sign-in ran") + assert.Empty(t, recorder.prompts(), "no page was shown, so no prompt is recorded") + require.NoError(t, p.finish(t)) +} + func TestAuthenticateReopensPageAfterTimeout(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { From bd5b8a2667d90fe78b324ea838e83cf64ad86394 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:48:27 +0000 Subject: [PATCH 04/43] Keep one set of login steps on an MCP authentication failure The MCP sign-in commands set their config for the telemetry upload, which also turns on root's auth enrichment. An AUTH_FAILED error already names the login command for its profile, so enrichment now leaves such an error as it is, and the user gets one set of next steps. Co-authored-by: Isaac --- cmd/aitools/mcp_login_test.go | 17 +++++++++++++++++ libs/aitools/mcp/binding.go | 6 ++++++ libs/auth/error.go | 10 ++++++++++ libs/auth/error_test.go | 27 +++++++++++++++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index dff27c19831..b041a502b86 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -20,6 +20,7 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" @@ -116,6 +117,22 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { assert.Contains(t, err.Error(), "Run: databricks auth login --profile logfood") } +// TestMcpLoginRejectedTokenHasOneSetOfNextSteps checks the error that root prints for a rejected token after login has +// its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. +func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { + home := withHome(t) + server := loginTestServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-expired\n") + + out := mcpSignInOutput{Service: "system.ai.github"} + err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogin, &out) + require.Error(t, err) + require.NotNil(t, out.cfg, "the client exists, so root enriches errors") + enriched := auth.EnrichAuthError(t.Context(), out.cfg, err) + assert.Equal(t, err.Error(), enriched.Error()) + assert.Equal(t, 0, strings.Count(enriched.Error(), "Next steps:")) +} + // TestMcpLogoutUnknownServiceIsAnError checks that logout of a service the profile cannot see fails, rather than // reporting that the caller is not signed in, so that a typo in the name is seen. func TestMcpLogoutUnknownServiceIsAnError(t *testing.T) { diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 78fb4dc52b0..173ffbabb14 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -60,6 +60,12 @@ func (e *BindError) Unwrap() error { return e.cause } +// NextStepsIncluded reports whether the message already names the next steps: an AUTH_FAILED message names the login +// command for the profile. The root command does not add a second set of steps to it. +func (e *BindError) NextStepsIncluded() bool { + return e.Category == BindAuthFailed +} + // Bind checks that the profile still names the host and workspace recorded for the service, // then authenticates with the profile's own credentials. It never reads stdin, prompts, or // loads a bundle. It returns the resolved config and the matching state entry, which is nil diff --git a/libs/auth/error.go b/libs/auth/error.go index fdb5f256f43..5e5b6369c24 100644 --- a/libs/auth/error.go +++ b/libs/auth/error.go @@ -82,6 +82,13 @@ func RewriteAuthError(ctx context.Context, host, accountId, profile string, err return false, err } +// nextStepsIncluded is implemented by an error whose message already gives the next steps for the failure, such as the +// login command of an MCP server. EnrichAuthError leaves such an error as it is, so the user is given one set of steps. +type nextStepsIncluded interface { + error + NextStepsIncluded() bool +} + // EnrichAuthError appends identity context and remediation steps to 401/403 API errors. // For non-API errors or other status codes, the original error is returned unchanged. func EnrichAuthError(ctx context.Context, cfg *config.Config, err error) error { @@ -92,6 +99,9 @@ func EnrichAuthError(ctx context.Context, cfg *config.Config, err error) error { if apiErr.StatusCode != http.StatusUnauthorized && apiErr.StatusCode != http.StatusForbidden { return err } + if steps, ok := errors.AsType[nextStepsIncluded](err); ok && steps.NextStepsIncluded() { + return err + } var b strings.Builder diff --git a/libs/auth/error_test.go b/libs/auth/error_test.go index ce8f0df1b63..b96a0386405 100644 --- a/libs/auth/error_test.go +++ b/libs/auth/error_test.go @@ -63,6 +63,33 @@ func TestEnrichAuthError_PreservesOriginalError(t *testing.T) { assert.Equal(t, "PERMISSION_DENIED", unwrapped.ErrorCode) } +// stepsIncludedError is an authentication failure whose message already gives the next steps. +type stepsIncludedError struct { + cause error +} + +func (e *stepsIncludedError) Error() string { + return "credential rejected. Run: databricks auth login --profile test" +} + +func (e *stepsIncludedError) Unwrap() error { + return e.cause +} + +func (e *stepsIncludedError) NextStepsIncluded() bool { + return true +} + +func TestEnrichAuthErrorLeavesErrorWithNextStepsAsIs(t *testing.T) { + cfg := &config.Config{Profile: "test", Host: "https://example.com", AuthType: "pat"} + original := &stepsIncludedError{cause: &apierr.APIError{StatusCode: 401, Message: "Unauthorized"}} + assert.Equal(t, original, EnrichAuthError(t.Context(), cfg, original)) + + // An API error without a message of its own is still enriched. + plain := &apierr.APIError{StatusCode: 401, Message: "Unauthorized"} + assert.Contains(t, EnrichAuthError(t.Context(), cfg, plain).Error(), "Next steps:") +} + func TestEnrichAuthError(t *testing.T) { tests := []struct { name string From b421db08e22328b2a011e1e6676a34734ef0f460 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:50:20 +0000 Subject: [PATCH 05/43] Resolve the remove upload's auth after the local edit remove resolved the profile before it touched any file, so an offline run could wait on the network first, even with telemetry disabled. The edit now runs first; the upload's profile is resolved afterwards, with a three second budget, and only when telemetry is on. Co-authored-by: Isaac --- cmd/aitools/mcp_remove.go | 3 +-- cmd/aitools/mcp_remove_test.go | 17 +++++++++++++++++ cmd/aitools/mcp_telemetry.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 14ad26c9db9..bbce687b4b2 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -100,14 +100,13 @@ removes one.`, databricks aitools mcp remove --profile logfood --agents cursor databricks aitools mcp remove --all`, Args: cobra.NoArgs, - // Resolve auth best-effort so telemetry can upload; see tryConfigureAuth. - PreRunE: tryConfigureAuth, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) out := &removeOutput{Removed: []removeRemoved{}, Skipped: []removeSkipped{}} runErr := runMcpRemove(ctx, cmd, f, jsonMode, out) + configureUploadAuth(cmd) logMcpEvent(ctx, protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), Operation: protos.AitoolsMcpOperationRemove, diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 19dbbe257bb..6c669ac3740 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -179,6 +179,23 @@ func TestMcpRemoveDeletesOwnedEntriesAndPrunesMissing(t *testing.T) { assert.Equal(t, "github-dogfood", state.Servers[0].Name) } +// TestMcpRemoveMakesNoRequestWhenTelemetryIsOff checks that remove edits its agent config without a request to the +// profile's host. Nothing is uploaded when telemetry is off, so no auth is resolved for an upload. +func TestMcpRemoveMakesNoRequestWhenTelemetryIsOff(t *testing.T) { + home := withHome(t) + t.Setenv("DATABRICKS_CLI_DISABLE_TELEMETRY", "1") + server := noRequestServer(t) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = oauth-m2m\nclient_id = id\nclient_secret = secret\n") + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + genie := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", server.URL) + saveState(t, genie) + + _, _, err := runMcpCmd(t, "remove", "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood") +} + func TestMcpRemoveSkipsEntryChangedOutsideCLI(t *testing.T) { home := withHome(t) claude := agents.ByName("claude-code") diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index ad05d2f6646..f23497a01d7 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -3,14 +3,47 @@ package aitools import ( "context" "strings" + "time" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" ) +// uploadAuthBudget bounds the resolution of the profile that authenticates a telemetry upload. +const uploadAuthBudget = 3 * time.Second + +// configureUploadAuth sets the config that authenticates the exit-time upload of a command that edits local files +// only. It runs after the edit, so an offline run does not wait on the network before it edits a file, and it does +// nothing when telemetry is disabled. The profile is the one the command would use without a flag; without one +// the upload is skipped. +func configureUploadAuth(cmd *cobra.Command) { + ctx := cmd.Context() + if telemetry.Disabled(ctx) { + return + } + configFile, _, err := addConfigFile(ctx) + if err != nil { + log.Debugf(ctx, "telemetry: no config file to upload with: %s", err) + return + } + profileName, err := resolveProfile(ctx, profileFlag(cmd)) + if err != nil { + log.Debugf(ctx, "telemetry: no profile to upload with: %s", err) + return + } + cfg, err := mcp.BoundConfigWithin(profileName, configFile, uploadAuthBudget) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) + return + } + cmd.SetContext(cmdctx.SetConfigUsed(ctx, cfg)) +} + // mcpServiceTypes maps the system.ai leaves in the CLI's service table to their telemetry enum. var mcpServiceTypes = map[string]protos.AitoolsMcpService{ "genie_one_mcp": protos.AitoolsMcpServiceGenieOne, From 80fe25b0c19a45eff31aebe32856d6f7c2fdd5f8 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:55:08 +0000 Subject: [PATCH 06/43] Log serve's start after its checks, and keep the failed start's upload PROXY_STARTED was logged before the startup checks, so a start the lookup rejected uploaded as a clean start. It is now logged once the checks have run, as AUTH_FAILED when they fail. A binding that fails AUTH_FAILED uploads with its profile, whose host the binding has already checked. A profile whose host changed still uploads nothing, since the binding never contacts a host the record does not name. A rejected refresh token clears the upload config, so the exit upload does not ask the token endpoint again. Co-authored-by: Isaac --- .../mcp-serve-rejected/.databrickscfg | 3 +++ .../telemetry/mcp-serve-rejected/out.test.toml | 1 + .../telemetry/mcp-serve-rejected/output.txt | 18 ++++++++++++++++++ .../telemetry/mcp-serve-rejected/script | 8 ++++++++ .../telemetry/mcp-serve-rejected/test.toml | 6 ++++++ .../cmd/aitools/telemetry/mcp-serve/output.txt | 3 +++ .../cmd/aitools/telemetry/mcp-serve/script | 2 +- cmd/aitools/mcp_serve.go | 14 +++++++++++++- cmd/aitools/mcp_serve_test.go | 4 ++++ cmd/aitools/mcp_telemetry.go | 13 ++++++++++--- libs/cmdctx/config_used.go | 6 ++++++ libs/cmdctx/config_used_test.go | 4 ++++ 12 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt new file mode 100644 index 00000000000..18e5860dd73 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt @@ -0,0 +1,18 @@ + +=== serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED + +>>> musterr [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code +No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] +cannot read MCP service system.ai.genie_one_mcp: Invalid access token. +Run: databricks auth login --profile logfood +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GENIE_ONE", + "error_category": "AUTH_FAILED" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script new file mode 100644 index 00000000000..0de703dfbc0 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script @@ -0,0 +1,8 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg +export BROWSER=none + +title "serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED" +trace musterr $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < /dev/null > /dev/null +print_requests.py //telemetry-ext | jq '.body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml new file mode 100644 index 00000000000..c081d32cf7a --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml @@ -0,0 +1,6 @@ +# The startup lookup of the service is rejected, so serve exits 2 before it serves. Its PROXY_STARTED event records the +# failure as AUTH_FAILED, and the upload that follows carries it. +[[Server]] +Pattern = "GET /api/2.1/unity-catalog/mcp-services/{name}" +Response.StatusCode = 401 +Response.Body = '{"error_code": "UNAUTHENTICATED", "message": "Invalid access token."}' diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt index 08586c94def..5ce1d986506 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -13,6 +13,9 @@ === A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with +>>> musterr [CLI] aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex +MCP server system.ai.genie_one_mcp needs Databricks profile "gone", which does not exist in [TEST_TMP_DIR]/.databrickscfg. Set it up with: databricks auth login --profile gone + === authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait { "calling_agent": "CLAUDE_CODE", diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script index fdceb646167..5b30a8e7ddc 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -23,7 +23,7 @@ $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --age mcp_events title "A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with" -errcode $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null 2> /dev/null +trace musterr $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null mcp_events title "authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait" diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 005805e8def..713df4c42ea 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -230,13 +230,19 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { Stderr: cmd.ErrOrStderr(), }) if err != nil { + // An AUTH_FAILED binding has passed the host check, so its profile may authenticate the upload. A profile whose + // host changed is not used: Bind never contacts a host the record does not name. + if errorCategory(err) == protos.AitoolsErrorCategoryAuthFailed { + if configFile, _, cerr := addConfigFile(cmd.Context()); cerr == nil { + setUploadConfig(cmd, profileName, configFile) + } + } logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveBindFailure(cmd, profileName, err) } // The bound profile authenticates the exit-time telemetry upload. cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), cfg)) - logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, nil)) // A server pinned to a config file names that file in its hints, as add and login do. pinnedFile := mcp.PinnedFile(entry, cfg.ConfigFile) @@ -261,6 +267,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. if refreshErr := refresh.failure(); refreshErr != nil { + cmd.SetContext(cmdctx.ClearConfigUsed(cmd.Context())) return serveFailure(cmd, refreshErr, serveBindStatus) } if ctx.Err() != nil { @@ -268,8 +275,10 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return nil } // serveChallenge fails only on an auth failure (see lookupAtStart), which exits serveBindStatus. + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveFailure(cmd, err, serveBindStatus) } + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, nil)) return serveProxy(ctx, cmd, refresh, proxy.Options{ In: cmd.InOrStdin(), @@ -477,6 +486,9 @@ func serveProxy(ctx context.Context, cmd *cobra.Command, refresh *refreshGuard, // stdin error after it is a consequence. The final line always names the rejection, as the bridge may not have logged it. func proxyResult(cmd *cobra.Command, refresh *refreshGuard, runErr error) error { if refreshErr := refresh.failure(); refreshErr != nil { + // The token was rejected or the binding changed, so the upload at exit would only ask the token endpoint again or + // use the re-pointed profile. + cmd.SetContext(cmdctx.ClearConfigUsed(cmd.Context())) fmt.Fprintf(cmd.ErrOrStderr(), "Error: stopped after %s for profile \"%s\"\n", stopReason(refreshErr), refresh.profile) return exitAlreadyPrinted(refreshErr, serveBindStatus) } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index fd2082308ed..e3c3cbcd90e 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -28,6 +28,7 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth/storage" "github.com/databricks/cli/libs/auth/u2m" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" @@ -607,11 +608,14 @@ func TestProxyResultRefreshFailureWinsOverStdinError(t *testing.T) { } require.Error(t, g.authenticateRequest(&http.Request{Header: http.Header{}})) cmd := NewMcpServeCmd() + cmd.SetContext(cmdctx.SetConfigUsed(t.Context(), &config.Config{Host: "https://adb-1.test"})) var stderr bytes.Buffer cmd.SetErr(&stderr) err := proxyResult(cmd, g, errors.New("read stdin: input/output error")) + // The exit-time upload would ask the rejected token's endpoint again, so it has no config. + assert.False(t, cmdctx.HasConfigUsed(cmd.Context())) assert.ErrorIs(t, err, root.ErrAlreadyPrinted) assert.Equal(t, serveBindStatus, root.ExitStatus(err)) assert.ErrorIs(t, err, cause) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index f23497a01d7..5db01724e1c 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -36,6 +36,13 @@ func configureUploadAuth(cmd *cobra.Command) { log.Debugf(ctx, "telemetry: no profile to upload with: %s", err) return } + setUploadConfig(cmd, profileName, configFile) +} + +// setUploadConfig sets the config of profileName in configFile for the exit-time upload, resolved within +// uploadAuthBudget. A profile that does not resolve uploads nothing. +func setUploadConfig(cmd *cobra.Command, profileName, configFile string) { + ctx := cmd.Context() cfg, err := mcp.BoundConfigWithin(profileName, configFile, uploadAuthBudget) if err != nil { log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) @@ -215,9 +222,9 @@ func signOutServerResult(service, state string, err error) protos.AitoolsMcpServ return r } -// proxyStartedEvent is the PROXY_STARTED event of serve. err is the binding failure, or nil when the -// proxy is about to serve. The event only uploads when the bound profile resolved, so a failed -// binding usually leaves it unsent. +// proxyStartedEvent is the PROXY_STARTED event of serve, logged once the startup checks have run. err is the failure +// of a check, or nil when the proxy is about to serve. The event uploads only when its profile resolved: a failed +// binding with no profile to upload with leaves it unsent. func proxyStartedEvent(agent protos.AitoolsAgentType, service string, err error) protos.AitoolsMcpEvent { r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} if err != nil { diff --git a/libs/cmdctx/config_used.go b/libs/cmdctx/config_used.go index 40595ef8aa9..ed20650aea1 100644 --- a/libs/cmdctx/config_used.go +++ b/libs/cmdctx/config_used.go @@ -18,6 +18,12 @@ func ConfigUsed(ctx context.Context) *config.Config { return cfg } +// ClearConfigUsed returns a context with no config used, as if SetConfigUsed had not run: the telemetry upload is +// skipped and auth errors are not enriched. +func ClearConfigUsed(ctx context.Context) context.Context { + return context.WithValue(ctx, configUsedKey, nil) +} + func HasConfigUsed(ctx context.Context) bool { _, ok := ctx.Value(configUsedKey).(*config.Config) return ok diff --git a/libs/cmdctx/config_used_test.go b/libs/cmdctx/config_used_test.go index f99febf8046..8a0724ceb76 100644 --- a/libs/cmdctx/config_used_test.go +++ b/libs/cmdctx/config_used_test.go @@ -27,6 +27,10 @@ func TestCommandConfigUsed(t *testing.T) { // The config should have the correct configuration. assert.Equal(t, "https://test.test", ConfigUsed(ctx).Host) + // Clearing the config leaves none, as if it had never been set. + assert.False(t, HasConfigUsed(ClearConfigUsed(ctx))) + assert.True(t, HasConfigUsed(ctx)) + // Second call should update the config used. cfg2 := &config.Config{ Host: "https://test2.test", From e97355c8cde8ca3a87365279355401de2d88b0a0 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:57:39 +0000 Subject: [PATCH 07/43] Log only events with a reached server, and count profiles by config file Login and logout log nothing when no server was reached, since their upload has no config. A sign-in state that was never learned is left unset rather than PENDING. A remove that only drops records logs its REMOVE, since the state changes. add --login logs SIGN_IN UNCHANGED when the credential was already active and the question was skipped. The bound profile count keys profiles by config file as well as name, as list does. Co-authored-by: Isaac --- .../aitools/telemetry/mcp-remove/output.txt | 15 ++++++++ .../mcp-remove/record-only.state.json | 20 ++++++++++ .../cmd/aitools/telemetry/mcp-remove/script | 6 +++ cmd/aitools/mcp_add.go | 7 +--- cmd/aitools/mcp_login.go | 5 ++- cmd/aitools/mcp_remove.go | 6 ++- cmd/aitools/mcp_telemetry.go | 31 +++++++++++---- cmd/aitools/mcp_telemetry_test.go | 38 ++++++++++++++++--- 8 files changed, 109 insertions(+), 19 deletions(-) create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt index 6ab420b2562..d470471158e 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -68,3 +68,18 @@ ], "bound_profile_count": 0 } + +=== A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound + +>>> [CLI] aitools mcp remove --names dbsql-logfood --profile logfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "DBSQL", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json new file mode 100644 index 00000000000..e069ce0fcac --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json @@ -0,0 +1,20 @@ +{ + "schema_version": 3, + "mcp": { + "servers": [ + { + "name": "dbsql-logfood", + "service": "system.ai.dbsql", + "profile": "logfood", + "host": "https://adb-1.test", + "workspace_id": "", + "sign_in_required": false, + "command": "", + "config_file": "", + "agents": {}, + "installed_at": "2026-01-01T00:00:00Z", + "cli_version": "0.0.0" + } + ] + } +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index 3d2481d9f43..fe054c38ce8 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -33,3 +33,9 @@ title "An entry already gone from the agent config is SKIPPED, and the state rec jq 'del(.mcpServers["genie_one_mcp-logfood"])' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null mcp_events + +title "A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound" +mkdir -p home/.databricks/aitools/skills +cp record-only.state.json home/.databricks/aitools/skills/.state.json +trace $CLI aitools mcp remove --names dbsql-logfood --profile logfood --output json > /dev/null +mcp_events diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index d73ea78847c..a01803a059c 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -1258,6 +1258,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp if signedIn(ctx, w, plans[i].service.FullName) { server.SignIn = &addSignInOutput{State: signInStateActive} reportSignedIn(errOut, plans[i].service, req.profile, true) + logMcpEvent(ctx, addSignInEvent(server.Service, signInStateActive, true, nil)) continue } yes, err := promptSignInNow(server.Service) @@ -1278,11 +1279,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp state, alreadySignedIn, _, err = signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) } server.SignIn = &addSignInOutput{State: state} - logMcpEvent(ctx, protos.AitoolsMcpEvent{ - CallingAgent: callingAgentType(), - Operation: protos.AitoolsMcpOperationSignIn, - Servers: []protos.AitoolsMcpServerResult{signInServerResult(server.Service, state, alreadySignedIn, err)}, - }) + logMcpEvent(ctx, addSignInEvent(server.Service, state, alreadySignedIn, err)) if err != nil { server.SignIn.ErrorCategory = errorCategory(err) server.SignIn.Error = err.Error() diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index 9307ed94013..5fd8821dd6d 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -106,7 +106,10 @@ func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAct if out.cfg != nil { cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), out.cfg)) } - logMcpEvent(ctx, signInEvent(operation, out, runErr)) + // The event uploads with the profile's config, so a run that reached no server logs nothing. + if out.cfg != nil { + logMcpEvent(ctx, signInEvent(operation, out, runErr)) + } if !jsonMode { return runErr } diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index bbce687b4b2..e7b078fbf5f 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -264,7 +264,11 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso removed.Agents = append(removed.Agents, agentName) drops = append(drops, recordDrop{name: e.Name, agent: agentName}) } - // A server with no agent left in state is removed from state too, so it is reported as removed. + // A server with no agent left in state is removed from state too, so it is reported as removed. Only its + // record goes, which is a change to the state. + if len(e.Agents) == 0 { + out.results = append(out.results, protos.AitoolsMcpServerResult{Service: mcpServiceType(e.Service), Outcome: protos.AitoolsTargetOutcomeChanged}) + } if len(removed.Agents) > 0 || len(e.Agents) == 0 { out.Removed = append(out.Removed, removed) } diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 5db01724e1c..e12c166ac39 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -111,15 +111,20 @@ func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) { log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) return } - event.BoundProfileCount = boundProfileCount(state) + event.BoundProfileCount = boundProfileCount(ctx, state) telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsMcpEvent: &event}) } -// boundProfileCount returns the number of distinct profiles that have servers installed. -func boundProfileCount(state *mcp.State) int64 { - profiles := map[string]bool{} +// boundProfileCount returns the number of distinct profiles that have servers installed. A profile is its name in its +// config file, as list groups them, so DEFAULT in two config files counts twice. +func boundProfileCount(ctx context.Context, state *mcp.State) int64 { + profiles := map[profileKey]bool{} for _, e := range state.Servers { - profiles[e.Profile] = true + path, err := mcp.ResolveConfigFile(ctx, e.ConfigFile) + if err != nil { + path = e.ConfigFile + } + profiles[profileKey{profile: e.Profile, path: path}] = true } return int64(len(profiles)) } @@ -178,10 +183,12 @@ func removeServerResult(service, agent string, outcome removeOutcome, err error) return r } -// signInStateOf maps a sign-in state name of login to its telemetry state. Anything but ACTIVE or -// NOT_REQUIRED is still PENDING. +// signInStateOf maps a sign-in state name of login to its telemetry state. A run that never learned the state leaves it +// unset. Anything but ACTIVE or NOT_REQUIRED is still PENDING. func signInStateOf(state string) protos.AitoolsMcpSignInState { switch state { + case "": + return "" case signInStateActive: return protos.AitoolsMcpSignInStateActive case signInStateNotRequired: @@ -206,6 +213,16 @@ func signInServerResult(service, state string, alreadySignedIn bool, err error) return r } +// addSignInEvent is the SIGN_IN event of one provider sign-in under add --login. A sign-in that found the caller +// already signed in is UNCHANGED. +func addSignInEvent(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpEvent { + return protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{signInServerResult(service, state, alreadySignedIn, err)}, + } +} + // signOutServerResult returns the result of a logout. Logout does not read whether the service needs // sign-in, so the sign-in state is not set. func signOutServerResult(service, state string, err error) protos.AitoolsMcpServerResult { diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 11cb852b8ef..046f7083ea4 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -3,6 +3,7 @@ package aitools import ( "encoding/json" "errors" + "path/filepath" "testing" "github.com/databricks/cli/libs/aitools/mcp" @@ -128,6 +129,15 @@ func TestSignInServerResult(t *testing.T) { state: signInStateNotRequired, want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateNotRequired}, }, + { + // A run that never learned the state, such as a read that failed, reports none. + name: "state not learned", + err: failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "not found"), + want: protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryMCPServiceNotFound, + }, + }, { name: "timed out", state: signInStateTimeout, @@ -193,13 +203,31 @@ func TestSignInPromptedEvent(t *testing.T) { } func TestBoundProfileCount(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "first.cfg") + second := filepath.Join(dir, "second.cfg") state := &mcp.State{Servers: []mcp.Entry{ - {Profile: "logfood", Service: "system.ai.github"}, - {Profile: "logfood", Service: "system.ai.genie_one_mcp"}, - {Profile: "dogfood", Service: "system.ai.github"}, + {Profile: "logfood", Service: "system.ai.github", ConfigFile: first}, + {Profile: "logfood", Service: "system.ai.genie_one_mcp", ConfigFile: first}, + // DEFAULT in two config files is two profiles, as list groups them. + {Profile: "DEFAULT", Service: "system.ai.github", ConfigFile: first}, + {Profile: "DEFAULT", Service: "system.ai.github", ConfigFile: second}, }} - assert.Equal(t, int64(2), boundProfileCount(state)) - assert.Equal(t, int64(0), boundProfileCount(&mcp.State{})) + assert.Equal(t, int64(3), boundProfileCount(t.Context(), state)) + assert.Equal(t, int64(0), boundProfileCount(t.Context(), &mcp.State{})) +} + +func TestAddSignInEvent(t *testing.T) { + // An ACTIVE credential that the add found signed in is UNCHANGED, as add --login reports it. + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeUnchanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }}, + }, addSignInEvent("system.ai.github", signInStateActive, true, nil)) } // The event carries enums and counts only. A custom service and the host of its profile must not From a5eca7cf51255dc7c0fac95b53b588bf0399c74e Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:58:17 +0000 Subject: [PATCH 08/43] Derive the telemetry service enum from the known system.ai table mcpServiceType parsed the system.ai schema name itself and repeated the table's leaves. It now uses the shared schema constant and the known table, so an unknown system.ai leaf is OTHER_SYSTEM and a service outside system.ai is CUSTOM. A test checks that every known leaf has its own enum. Co-authored-by: Isaac --- cmd/aitools/mcp_telemetry.go | 8 ++++---- cmd/aitools/mcp_telemetry_test.go | 10 ++++++++++ libs/aitools/mcp/known.go | 11 ++++++++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index e12c166ac39..c3c7b32ecca 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -71,13 +71,13 @@ var mcpServiceTypes = map[string]protos.AitoolsMcpService{ // services outside system.ai are CUSTOM. func mcpServiceType(fullName string) protos.AitoolsMcpService { parts := strings.Split(fullName, ".") - if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], "system.ai") { + if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], mcp.SystemAILocation) { return protos.AitoolsMcpServiceCustom } - if t, ok := mcpServiceTypes[strings.ToLower(parts[2])]; ok { - return t + if mcp.KnownOf(fullName) == nil { + return protos.AitoolsMcpServiceOtherSystem } - return protos.AitoolsMcpServiceOtherSystem + return mcpServiceTypes[strings.ToLower(parts[2])] } // callingAgentType returns the agent that drives the CLI, Unspecified when none is detected. diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 046f7083ea4..df0d31c894e 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -40,6 +40,16 @@ func TestMcpServiceType(t *testing.T) { } } +// Every service in the CLI's known table has its own enum, so a known service never reports OTHER_SYSTEM. A new table +// entry needs a new value in mcpServiceTypes. +func TestMcpServiceTypeCoversKnownServices(t *testing.T) { + for _, leaf := range mcp.KnownLeaves() { + assert.Contains(t, mcpServiceTypes, leaf, "system.ai.%s has no telemetry enum: add it to mcpServiceTypes", leaf) + assert.NotEqualf(t, protos.AitoolsMcpServiceOtherSystem, mcpServiceType(mcp.SystemAILocation+"."+leaf), + "system.ai.%s has no telemetry enum", leaf) + } +} + func TestAddServerResults(t *testing.T) { servers := []addServerOutput{ { diff --git a/libs/aitools/mcp/known.go b/libs/aitools/mcp/known.go index 867d2b0ff36..10bf6294947 100644 --- a/libs/aitools/mcp/known.go +++ b/libs/aitools/mcp/known.go @@ -1,6 +1,10 @@ package mcp -import "strings" +import ( + "maps" + "slices" + "strings" +) // SystemAILocation is the schema that ships the Databricks managed MCP services. const SystemAILocation = "system.ai" @@ -61,6 +65,11 @@ var knownSystemAI = map[string]KnownService{ }, } +// KnownLeaves returns the leaf names of the system.ai services in the CLI's table, sorted. +func KnownLeaves() []string { + return slices.Sorted(maps.Keys(knownSystemAI)) +} + // KnownOf returns the CLI's description of a service, or nil when it is not a system.ai service in the table. The leaf // is compared without case, as Unity Catalog names are. func KnownOf(fullName string) *KnownService { From 73ce890db50d7acf5e078ac5e4e8d2ff8c7c8170 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:00:18 +0000 Subject: [PATCH 09/43] Tidy the telemetry acceptance scripts and the fake telemetry endpoint The four telemetry scripts share one mcp_events helper that filters the uploads with print_requests.py. Their API posts carry MSYS_NO_PATHCONV so Windows keeps the path, the required failures use musterr, the invalid Codex config is a committed input, and the CLI calls are traced. The fake /telemetry-ext reports an unreadable batch as an error, as its neighbors do. Co-authored-by: Isaac --- .../telemetry/mcp-add/codex-config.toml | 1 + .../cmd/aitools/telemetry/mcp-add/output.txt | 6 ++++- .../cmd/aitools/telemetry/mcp-add/script | 20 +++++----------- .../aitools/telemetry/mcp-remove/output.txt | 10 ++++++++ .../cmd/aitools/telemetry/mcp-remove/script | 20 +++++----------- .../telemetry/mcp-serve-rejected/script | 2 +- .../aitools/telemetry/mcp-serve/output.txt | 3 +++ .../cmd/aitools/telemetry/mcp-serve/script | 16 ++++--------- .../aitools/telemetry/mcp-sign-in/output.txt | 8 +++++++ .../cmd/aitools/telemetry/mcp-sign-in/script | 24 +++++++------------ .../cmd/aitools/telemetry/script.prepare | 8 +++++++ 11 files changed, 60 insertions(+), 58 deletions(-) create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml b/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml new file mode 100644 index 00000000000..8758ff120ff --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml @@ -0,0 +1 @@ +not = [valid diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/output.txt b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt index b7ea889e8b8..132249dbdea 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-add/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt @@ -1,5 +1,7 @@ === A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", @@ -33,6 +35,8 @@ } === The same servers again are re-asserted: UNCHANGED + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", @@ -67,7 +71,7 @@ === An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID -Exit code: 1 +>>> musterr [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/script b/acceptance/cmd/aitools/telemetry/mcp-add/script index 2280d098810..6cf6f91e92a 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-add/script +++ b/acceptance/cmd/aitools/telemetry/mcp-add/script @@ -3,28 +3,20 @@ mkdir -p home/.claude home/.codex envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg -# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. -mcp_events() { - if [ -f "$OUT_REQUESTS" ]; then - jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" - rm -f "$OUT_REQUESTS" - fi -} - # Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null rm -f "$OUT_REQUESTS" title "A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING" -$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null mcp_events title "The same servers again are re-asserted: UNCHANGED" -$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null mcp_events title "An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID" -printf 'not = [valid\n' > home/.codex/config.toml -errcode $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json > /dev/null +cp codex-config.toml home/.codex/config.toml +trace musterr $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json > /dev/null mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt index d470471158e..4f85fef980b 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -1,5 +1,7 @@ === Installed for two profiles: the bound profile count is 2 after the second add + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", @@ -13,6 +15,8 @@ ], "bound_profile_count": 1 } + +>>> [CLI] aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", @@ -28,6 +32,8 @@ } === remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound + +>>> [CLI] aitools mcp remove --profile dogfood --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "REMOVE", @@ -42,6 +48,8 @@ } === An entry changed outside this CLI is left in place: SKIPPED + +>>> [CLI] aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "REMOVE", @@ -56,6 +64,8 @@ } === An entry already gone from the agent config is SKIPPED, and the state record goes + +>>> [CLI] aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "REMOVE", diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index fe054c38ce8..ad266636d8a 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -3,35 +3,27 @@ mkdir -p home/.claude envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg -# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. -mcp_events() { - if [ -f "$OUT_REQUESTS" ]; then - jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" - rm -f "$OUT_REQUESTS" - fi -} - -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null rm -f "$OUT_REQUESTS" title "Installed for two profiles: the bound profile count is 2 after the second add" -$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null mcp_events -$CLI aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +trace $CLI aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null mcp_events title "remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound" -$CLI aitools mcp remove --profile dogfood --output json > /dev/null +trace $CLI aitools mcp remove --profile dogfood --output json > /dev/null mcp_events title "An entry changed outside this CLI is left in place: SKIPPED" jq '.mcpServers["genie_one_mcp-logfood"].command = "/usr/bin/other"' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json -$CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null mcp_events title "An entry already gone from the agent config is SKIPPED, and the state record goes" jq 'del(.mcpServers["genie_one_mcp-logfood"])' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json -$CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null mcp_events title "A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound" diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script index 0de703dfbc0..25fb6292ef9 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script @@ -5,4 +5,4 @@ export BROWSER=none title "serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED" trace musterr $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < /dev/null > /dev/null -print_requests.py //telemetry-ext | jq '.body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt index 5ce1d986506..1bcd45d41b1 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -1,5 +1,8 @@ === serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes + +>>> [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code +No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] { "calling_agent": "CLAUDE_CODE", "operation": "PROXY_STARTED", diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script index 5b30a8e7ddc..2e865c20274 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -5,21 +5,13 @@ export DATABRICKS_CONFIG_FILE=.databrickscfg # No browser: the proxy prints the sign-in URL and does not open anything. export BROWSER=none export RETRY_MAX_ATTEMPTS=60 -# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. -mcp_events() { - if [ -f "$OUT_REQUESTS" ]; then - jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" - rm -f "$OUT_REQUESTS" - fi -} - # Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null rm -f "$OUT_REQUESTS" title "serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes" -$CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null 2> /dev/null +trace $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null mcp_events title "A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with" @@ -32,7 +24,7 @@ title "authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED retry --until '"id":2' cat serve.stdout.txt > /dev/null cat requests-authenticate.ndjson retry --until "Waiting for sign-in" cat serve.stderr.txt > /dev/null - $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null + MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null retry --until '"id":3' cat serve.stdout.txt > /dev/null cat requests-after.ndjson } | $CLI aitools mcp serve --profile logfood --service system.ai.github --agent claude-code > serve.stdout.txt 2> serve.stderr.txt diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt index d2b56d441f2..46253de58e8 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt @@ -1,5 +1,7 @@ === login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED + +>>> [CLI] aitools mcp login system.ai.genie_one_mcp --profile logfood system.ai.genie_one_mcp does not need sign-in. { "calling_agent": "TYPE_UNSPECIFIED", @@ -29,6 +31,8 @@ system.ai.genie_one_mcp does not need sign-in. } === login while already signed in: UNCHANGED + +>>> [CLI] aitools mcp login system.ai.github --profile logfood Already signed in to system.ai.github on profile logfood. Public repositories only until a repository owner installs https://github.com/apps/databricks-github-connector. { @@ -45,6 +49,8 @@ Public repositories only until a repository owner installs https://github.com/ap } === logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED + +>>> [CLI] aitools mcp logout system.ai.github --profile logfood Signed out of system.ai.github on profile logfood. { "calling_agent": "TYPE_UNSPECIFIED", @@ -57,6 +63,8 @@ Signed out of system.ai.github on profile logfood. ], "bound_profile_count": 0 } + +>>> [CLI] aitools mcp logout system.ai.github --profile logfood Not signed in to system.ai.github on profile logfood. { "calling_agent": "TYPE_UNSPECIFIED", diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script index 72025e3cde6..6ddd1bce499 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script @@ -6,47 +6,39 @@ export DATABRICKS_CONFIG_FILE=.databrickscfg # No browser: the CLI prints the sign-in URL and polls. export BROWSER=none export RETRY_MAX_ATTEMPTS=60 -# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. -mcp_events() { - if [ -f "$OUT_REQUESTS" ]; then - jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" - rm -f "$OUT_REQUESTS" - fi -} - # Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null rm -f "$OUT_REQUESTS" title "login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED" -$CLI aitools mcp login system.ai.genie_one_mcp --profile logfood > /dev/null +trace $CLI aitools mcp login system.ai.genie_one_mcp --profile logfood > /dev/null mcp_events title "login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE" $CLI aitools mcp login system.ai.github --profile logfood > login.txt 2>&1 & PID=$! retry --until "Waiting for sign-in" cat login.txt > /dev/null -$CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null wait $PID mcp_events rm login.txt title "login while already signed in: UNCHANGED" -$CLI aitools mcp login system.ai.github --profile logfood > /dev/null +trace $CLI aitools mcp login system.ai.github --profile logfood > /dev/null mcp_events title "logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED" -$CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +trace $CLI aitools mcp logout system.ai.github --profile logfood > /dev/null mcp_events -$CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +trace $CLI aitools mcp logout system.ai.github --profile logfood > /dev/null mcp_events title "add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE" $CLI aitools mcp add --profile logfood --names system.ai.github --agents claude-code --login > add.txt 2>&1 & PID=$! retry --until "Waiting for sign-in" cat add.txt > /dev/null -$CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null wait $PID mcp_events rm add.txt diff --git a/acceptance/cmd/aitools/telemetry/script.prepare b/acceptance/cmd/aitools/telemetry/script.prepare index f55e1cef710..6a9112e4947 100644 --- a/acceptance/cmd/aitools/telemetry/script.prepare +++ b/acceptance/cmd/aitools/telemetry/script.prepare @@ -15,3 +15,11 @@ print_telemetry() { + (del(.execution_context))' fi } + +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. The uploads are +# filtered by path with print_requests.py; jq reads only the body. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + print_requests.py //telemetry-ext | jq '.body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' + fi +} From de8a655645bc2a8866aa924a789160f4daa53100 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:01:31 +0000 Subject: [PATCH 10/43] Document the error categories of the server result and tidy its test The server result's doc now names the categories a proxy start and a SIGN_IN_PROMPTED carry. The user-content test uses slices and maps from the standard library, and its comment no longer claims a host check the test does not make. Co-authored-by: Isaac --- cmd/aitools/mcp_telemetry_test.go | 15 ++++----------- libs/telemetry/protos/aitools_mcp.go | 5 +++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index df0d31c894e..b78201e3e29 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -3,7 +3,9 @@ package aitools import ( "encoding/json" "errors" + "maps" "path/filepath" + "slices" "testing" "github.com/databricks/cli/libs/aitools/mcp" @@ -240,8 +242,7 @@ func TestAddSignInEvent(t *testing.T) { }, addSignInEvent("system.ai.github", signInStateActive, true, nil)) } -// The event carries enums and counts only. A custom service and the host of its profile must not -// appear anywhere in the payload. +// The event carries enums and counts only. A custom service's name must not appear anywhere in the payload. func TestMcpEventOmitsUserContent(t *testing.T) { event := protos.AitoolsMcpEvent{ CallingAgent: protos.AitoolsAgentTypeClaudeCode, @@ -257,17 +258,9 @@ func TestMcpEventOmitsUserContent(t *testing.T) { var got map[string]any require.NoError(t, json.Unmarshal(b, &got)) - assert.ElementsMatch(t, []string{"calling_agent", "operation", "servers", "bound_profile_count"}, keys(got)) + assert.ElementsMatch(t, []string{"calling_agent", "operation", "servers", "bound_profile_count"}, slices.Collect(maps.Keys(got))) assert.NotContains(t, string(b), "acme_catalog") assert.NotContains(t, string(b), "secret_schema") assert.NotContains(t, string(b), "internal_tool") assert.Contains(t, string(b), `"service":"CUSTOM"`) } - -func keys(m map[string]any) []string { - out := make([]string, 0, len(m)) - for k := range m { - out = append(out, k) - } - return out -} diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index c42dbc20285..f6a8d2f677b 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -63,8 +63,9 @@ const ( // AitoolsMcpServerResult is the result for one service, for one agent when the // operation writes agent config. Unset fields are omitted: an agent is not set -// for sign-in, an outcome is not set for proxy events, and an error category is -// set only when the target failed or was skipped. +// for sign-in, and an outcome is not set for proxy events. An error category is +// set when the target failed or was skipped, when a proxy start failed its +// checks (PROXY_STARTED), and on SIGN_IN_PROMPTED, where it is SIGN_IN_REQUIRED. type AitoolsMcpServerResult struct { Service AitoolsMcpService `json:"service"` Agent AitoolsAgentType `json:"agent,omitempty"` From 3c2b655979064aea1c78cd04adefc42023b4b56c Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:33:25 +0000 Subject: [PATCH 11/43] Authenticate telemetry uploads without refreshing a token The config of a telemetry upload that the command does not otherwise use never refreshes its token: the command may exit before a refresh finishes, and a rotated single-use refresh token would be lost. serve's session keeps its refreshing config, and its upload gets a config of its own. Co-authored-by: Isaac --- cmd/aitools/mcp_serve.go | 5 +++-- cmd/aitools/mcp_telemetry.go | 9 +++------ libs/aitools/mcp/binding.go | 19 +++++++++++++++++-- libs/aitools/mcp/binding_test.go | 15 +++++++++++++++ 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 713df4c42ea..6eebff64ea2 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -241,8 +241,9 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return serveBindFailure(cmd, profileName, err) } - // The bound profile authenticates the exit-time telemetry upload. - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), cfg)) + // The bound profile authenticates the exit-time telemetry upload. The session's own config refreshes its token, which + // the upload must not do, so the upload gets a config of its own. + setUploadConfig(cmd, profileName, cfg.ConfigFile) // A server pinned to a config file names that file in its hints, as add and login do. pinnedFile := mcp.PinnedFile(entry, cfg.ConfigFile) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index c3c7b32ecca..0f609ba3a42 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -3,7 +3,6 @@ package aitools import ( "context" "strings" - "time" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" @@ -14,9 +13,6 @@ import ( "github.com/spf13/cobra" ) -// uploadAuthBudget bounds the resolution of the profile that authenticates a telemetry upload. -const uploadAuthBudget = 3 * time.Second - // configureUploadAuth sets the config that authenticates the exit-time upload of a command that edits local files // only. It runs after the edit, so an offline run does not wait on the network before it edits a file, and it does // nothing when telemetry is disabled. The profile is the one the command would use without a flag; without one @@ -40,10 +36,11 @@ func configureUploadAuth(cmd *cobra.Command) { } // setUploadConfig sets the config of profileName in configFile for the exit-time upload, resolved within -// uploadAuthBudget. A profile that does not resolve uploads nothing. +// telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A profile that does not resolve +// uploads nothing. func setUploadConfig(cmd *cobra.Command, profileName, configFile string) { ctx := cmd.Context() - cfg, err := mcp.BoundConfigWithin(profileName, configFile, uploadAuthBudget) + cfg, err := mcp.BoundUploadConfig(profileName, configFile, telemetryAuthBudget) if err != nil { log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) return diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 173ffbabb14..79d473b01b8 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -11,6 +11,7 @@ import ( "time" "github.com/databricks/cli/libs/aitools/mcpconfig" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" @@ -116,14 +117,28 @@ func BoundConfig(profileName, configFile string) (*config.Config, error) { // budget of zero keeps the SDK's defaults. Resolving the profile runs on a context of its own, so a context deadline does // not reach it, and these budgets are what bound it (see profileMetadata.Load in cmd/auth). func BoundConfigWithin(profileName, configFile string, budget time.Duration) (*config.Config, error) { + return resolveBound(withinBudget(profileName, configFile, budget)) +} + +// BoundUploadConfig is BoundConfigWithin for a config that only a telemetry upload uses. Its credentials never refresh a +// token, since the command may exit before a refresh finishes, and a rotated single-use refresh token would be lost (see +// auth.CredentialsWithoutRefresh). It does not authenticate. +func BoundUploadConfig(profileName, configFile string, budget time.Duration) (*config.Config, error) { + cfg := withinBudget(profileName, configFile, budget) + cfg.Credentials = auth.CredentialsWithoutRefresh() + return resolveBound(cfg) +} + +// withinBudget is the config of profileName in configFile, with the SDK's request and retry budgets set to budget. +func withinBudget(profileName, configFile string, budget time.Duration) *config.Config { seconds := int((budget + time.Second - 1) / time.Second) - return resolveBound(&config.Config{ + return &config.Config{ Profile: profileName, ConfigFile: configFile, Loaders: []config.Loader{databrickscfg.ResolveNonAuthFromEnv, config.ConfigFile}, HTTPTimeoutSeconds: seconds, RetryTimeoutSeconds: seconds, - }) + } } // resolveBound resolves cfg, as BoundConfig does. diff --git a/libs/aitools/mcp/binding_test.go b/libs/aitools/mcp/binding_test.go index 3541917be0b..f98905fa28b 100644 --- a/libs/aitools/mcp/binding_test.go +++ b/libs/aitools/mcp/binding_test.go @@ -10,8 +10,10 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/shellquote" "github.com/stretchr/testify/assert" @@ -944,3 +946,16 @@ func TestConfigFileCommandUsesForwardSlashes(t *testing.T) { assert.Equal(t, "DATABRICKS_CONFIG_FILE="+filepath.ToSlash(path)+" databricks auth login --profile logfood", mcp.ConfigFileCommand(path, "databricks auth login --profile logfood")) } + +// The config of a telemetry upload never refreshes a token; the config a serve session binds keeps the default chain. +func TestBoundUploadConfigNeverRefreshes(t *testing.T) { + cfgPath := bindingTestConfig(t, bindingTestProfileContents(bindingTestToken)) + + upload, err := mcp.BoundUploadConfig(bindingTestProfile, cfgPath, time.Second) + require.NoError(t, err) + assert.Equal(t, auth.CredentialsWithoutRefresh(), upload.Credentials) + + session, err := mcp.BoundConfigWithin(bindingTestProfile, cfgPath, time.Second) + require.NoError(t, err) + assert.NotEqual(t, auth.CredentialsWithoutRefresh(), session.Credentials) +} From 5bd3941d5a49ea51cc72013f42d62277c18a5cc5 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:36:29 +0000 Subject: [PATCH 12/43] Resolve remove's telemetry auth only after it logs an event remove writes its output first, then resolves the auth of its upload, only when a server was touched. The upload is skipped when a server recorded for its profile no longer matches the profile's host, workspace or config file, so the token is not sent to a host the server was not set up against. The auth resolution is the one install uses, logged at debug level and bounded by the same budget. Co-authored-by: Isaac --- .../mcp-remove/record-only.state.json | 4 +- .../cmd/aitools/telemetry/mcp-remove/script | 2 +- cmd/aitools/mcp_remove.go | 76 ++++++++---- cmd/aitools/mcp_remove_test.go | 110 ++++++++++++++++++ cmd/aitools/mcp_telemetry.go | 38 +++--- 5 files changed, 184 insertions(+), 46 deletions(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json index e069ce0fcac..e40e76fff80 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json @@ -6,11 +6,11 @@ "name": "dbsql-logfood", "service": "system.ai.dbsql", "profile": "logfood", - "host": "https://adb-1.test", + "host": "$DATABRICKS_HOST", "workspace_id": "", "sign_in_required": false, "command": "", - "config_file": "", + "config_file": "$PWD/.databrickscfg", "agents": {}, "installed_at": "2026-01-01T00:00:00Z", "cli_version": "0.0.0" diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index ad266636d8a..ef98d35f619 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -28,6 +28,6 @@ mcp_events title "A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound" mkdir -p home/.databricks/aitools/skills -cp record-only.state.json home/.databricks/aitools/skills/.state.json +envsubst < record-only.state.json > home/.databricks/aitools/skills/.state.json trace $CLI aitools mcp remove --names dbsql-logfood --profile logfood --output json > /dev/null mcp_events diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index e7b078fbf5f..25cd0f640ca 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -13,6 +13,8 @@ import ( "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/aitools/mcp/agentconfig" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -35,6 +37,8 @@ type removeOutput struct { // results is one telemetry result per (service, agent) attempted. It is not in the JSON. results []protos.AitoolsMcpServerResult + // stateBefore is the state as it was read before the removal. It is not in the JSON. + stateBefore *mcp.State } type removeRemoved struct { @@ -106,28 +110,12 @@ removes one.`, out := &removeOutput{Removed: []removeRemoved{}, Skipped: []removeSkipped{}} runErr := runMcpRemove(ctx, cmd, f, jsonMode, out) - configureUploadAuth(cmd) - logMcpEvent(ctx, protos.AitoolsMcpEvent{ - CallingAgent: callingAgentType(), - Operation: protos.AitoolsMcpOperationRemove, - Servers: out.results, - }) - if !jsonMode { - return runErr - } - - // The document carries a top-level failure; per-agent failures are already in skipped. - if top := topLevelFailure(runErr); top != nil { - out.Error = top.Error() - out.ErrorCategory = errorCategory(top) + err := runErr + if jsonMode { + err = writeRemoveJSON(cmd, out, runErr) } - if err := renderJSON(cmd.OutOrStdout(), out); err != nil { - return errors.Join(runErr, err) - } - if runErr != nil { - return root.ErrAlreadyPrinted - } - return nil + logRemoveEvent(cmd, args, out) + return err }, } @@ -137,6 +125,51 @@ removes one.`, return cmd } +// writeRemoveJSON writes the JSON document of a run that ended with runErr and returns the command's error. +func writeRemoveJSON(cmd *cobra.Command, out *removeOutput, runErr error) error { + // The document carries a top-level failure; per-agent failures are already in skipped. + if top := topLevelFailure(runErr); top != nil { + out.Error = top.Error() + out.ErrorCategory = errorCategory(top) + } + if err := renderJSON(cmd.OutOrStdout(), out); err != nil { + return errors.Join(runErr, err) + } + if runErr != nil { + return root.ErrAlreadyPrinted + } + return nil +} + +// logRemoveEvent buffers the REMOVE event and sets the config its exit-time upload authenticates with. It runs after +// the output is written, and only when the run touched a server: a run that touched none has nothing to upload, so it +// resolves no auth. The auth resolves as install's does (startTelemetryAuth), within telemetryAuthBudget, and the upload +// is skipped when a server recorded for the profile no longer matches it (bindingDrifted). +func logRemoveEvent(cmd *cobra.Command, args []string, out *removeOutput) { + ctx := cmd.Context() + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationRemove, + Servers: out.results, + }) + if len(telemetry.BufferedEvents(ctx)) == 0 { + return + } + auth := startTelemetryAuth(cmd, args) + if auth == nil { + return + } + if !auth.wait() { + log.Debugf(ctx, "auth for telemetry not resolved within %s; not uploading telemetry", telemetryAuthBudget) + return + } + if auth.cfg != nil && bindingDrifted(ctx, out.stateBefore, auth.cfg) { + log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", auth.cfg.Profile) + return + } + auth.apply(cmd, false) +} + // runMcpRemove removes the chosen servers from the agent configs and records the result. Results are printed // before a state failure is returned. An agent failure fails the command when --agents named the agent, or when a // selected server was removed from no agent; otherwise it is reported in skipped. @@ -170,6 +203,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if err != nil { return err } + out.stateBefore = state var selected []mcp.Entry if interactive && !f.all && len(names) == 0 && profileName == "" { // An agent with no installed server is refused as it is without a terminal, before the picker offers nothing for it. diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 6c669ac3740..ddaf9d45f05 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -1,19 +1,30 @@ package aitools import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "maps" "os" "path/filepath" "slices" "testing" + "time" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/log/handler" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/config" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -993,3 +1004,102 @@ func claudeServerEntry(claudePath, name, service string) mcp.Entry { Agents: map[string]mcp.AgentRecord{agents.NameClaudeCode: {Path: claudePath, Scope: "user"}}, } } + +// runRemoveWithTelemetry runs `mcp remove` with args on a context that buffers telemetry, and returns the remove command, +// so a test can read the context it left, and its stdout. +// The logs are those of the command's logger at logLevel. +func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel slog.Level, args ...string) (*cobra.Command, error) { + t.Helper() + parent := newMCPTestRoot() + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + ctx = log.NewContext(ctx, slog.New(handler.NewFriendlyHandler(logs, &handler.Options{Level: logLevel}))) + ctx, _ = cmdio.NewTestContextWithStderr(ctx) + parent.SetOut(stdout) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs(append([]string{"mcp", "remove"}, args...)) + err := parent.ExecuteContext(ctx) + remove, _, findErr := parent.Find([]string{"mcp", "remove"}) + require.NoError(t, findErr) + return remove, err +} + +// A run that removes nothing logs no event, so it resolves no auth for an upload. +func TestMcpRemoveResolvesNothingWithoutAnEvent(t *testing.T) { + withHome(t) + resolves := 0 + setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { + resolves++ + return nil + }) + + var stdout, logs bytes.Buffer + _, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--all", "--output", "json") + require.NoError(t, err) + assert.Zero(t, resolves) +} + +// The upload is authenticated with the profile only when no server recorded for it has drifted: its host here is the one +// the server was set up against. The removal is done either way, and the output is written before the auth resolves. +func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + // Recorded against another host than the profile names now. + entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + saveState(t, entry) + + var stdout, logs bytes.Buffer + var outputWrittenFirst bool + setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { + outputWrittenFirst = stdout.Len() > 0 + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Profile: "logfood", Host: "https://adb-2.test", ConfigFile: filepath.Join(home, ".databrickscfg")})) + return nil + }) + + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + assert.True(t, outputWrittenFirst, "the auth resolves after the JSON is written") + assert.False(t, cmdctx.HasConfigUsed(remove.Context()), "a drifted profile is not used to upload") + assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood", "the removal is done") +} + +// A profile whose recorded servers still match is used for the upload. +func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + saveState(t, entry) + + setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Profile: "logfood", Host: "https://adb-1.test", ConfigFile: filepath.Join(home, ".databrickscfg")})) + return nil + }) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + require.True(t, cmdctx.HasConfigUsed(remove.Context())) + assert.Equal(t, "https://adb-1.test", cmdctx.ConfigUsed(remove.Context()).Host) +} + +// An auth resolution that fails, as it does offline, leaves nothing in the output: its logs are debug only. +func TestMcpRemoveOfflineResolutionWarnsNothing(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + + setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { + log.Warnf(cmd.Context(), "[hostmetadata] failed to fetch host metadata for https://adb-1.test") + return errors.New("cannot reach https://adb-1.test") + }) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelWarn, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + assert.False(t, cmdctx.HasConfigUsed(remove.Context())) + assert.Empty(t, logs.String()) + assert.Contains(t, stdout.String(), `"removed"`) +} diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 0f609ba3a42..e3b623f8282 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -10,31 +10,10 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" ) -// configureUploadAuth sets the config that authenticates the exit-time upload of a command that edits local files -// only. It runs after the edit, so an offline run does not wait on the network before it edits a file, and it does -// nothing when telemetry is disabled. The profile is the one the command would use without a flag; without one -// the upload is skipped. -func configureUploadAuth(cmd *cobra.Command) { - ctx := cmd.Context() - if telemetry.Disabled(ctx) { - return - } - configFile, _, err := addConfigFile(ctx) - if err != nil { - log.Debugf(ctx, "telemetry: no config file to upload with: %s", err) - return - } - profileName, err := resolveProfile(ctx, profileFlag(cmd)) - if err != nil { - log.Debugf(ctx, "telemetry: no profile to upload with: %s", err) - return - } - setUploadConfig(cmd, profileName, configFile) -} - // setUploadConfig sets the config of profileName in configFile for the exit-time upload, resolved within // telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A profile that does not resolve // uploads nothing. @@ -280,3 +259,18 @@ func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn }}, } } + +// bindingDrifted reports whether a server recorded for cfg's profile in state no longer matches the profile: its host or +// workspace changed, or its config file moved. The upload would send the profile's token to the host the profile names +// now, which the server was not set up against. It reads the config files only. +func bindingDrifted(ctx context.Context, state *mcp.State, cfg *config.Config) bool { + for _, e := range state.Servers { + if e.Profile != cfg.Profile { + continue + } + if _, err := mcp.CheckBinding(ctx, state, e.Service, cfg.Profile, cfg.ConfigFile); err != nil { + return true + } + } + return false +} From 0f0e1a01cebbac1247dbace22c247554cd693108 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:38:08 +0000 Subject: [PATCH 13/43] Resolve add, login and logout telemetry auth as install does The three commands no longer set their profile's config for the upload themselves. withTelemetryAuth resolves it in the background, as install does, and the run waits for it once it has logged an event. The events upload with the auth root resolves, so root's auth enrichment still applies to their errors; the MCP login hint stays the only set of next steps. Co-authored-by: Isaac --- cmd/aitools/mcp_add.go | 8 +++++--- cmd/aitools/mcp_login.go | 24 ++++++++++++------------ cmd/aitools/mcp_login_test.go | 8 +++++--- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index a01803a059c..91b35de3720 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -20,7 +20,6 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/browser" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" @@ -161,6 +160,7 @@ func (o *addOutput) slashPaths() { // NewMcpAddCmd returns `aitools mcp add`. func NewMcpAddCmd() *cobra.Command { var f mcpAddFlags + var auth *telemetryAuth cmd := &cobra.Command{ Use: "add", Short: "Install Databricks managed MCP servers into coding agents", @@ -183,6 +183,8 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) + // The events upload with the auth resolved for the profile, and a run that logs one waits for it. + auth.waitAtExit = true out := &addOutput{ Scope: installer.ScopeGlobal, @@ -221,6 +223,8 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), cmd.Flags().BoolVar(&f.force, "force", false, "Overwrite an existing server with the same name that this CLI did not write") cmd.Flags().BoolVar(&f.login, "login", false, "Sign in to each sign-in-required server that was installed, after the installs") cmd.Flags().BoolVar(&f.noLogin, "no-login", false, "Do not sign in; print the login commands for sign-in-required servers (default without a terminal; in a terminal, the default asks for each one)") + // Resolve auth best-effort so the telemetry can upload, as install does. + auth = withTelemetryAuth(cmd) return cmd } @@ -381,8 +385,6 @@ func runMcpAdd(ctx context.Context, cmd *cobra.Command, f mcpAddFlags, jsonMode return err } } - // The bound profile authenticates the exit-time telemetry upload. - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), w.Config)) for i := range plans { plans[i].service = picked[i] } diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index 5fd8821dd6d..ed92d9b0657 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -12,12 +12,10 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/browser" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" ) @@ -48,8 +46,6 @@ type mcpSignInOutput struct { ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` Error string `json:"error,omitempty"` - // cfg is the bound profile's config, which authenticates the exit-time telemetry upload. - cfg *config.Config // alreadySignedIn is set by login when the caller was signed in before it ran. alreadySignedIn bool } @@ -60,7 +56,8 @@ type mcpSignInAction func(ctx context.Context, errOut io.Writer, w *databricks.W // NewMcpLoginCmd returns `aitools mcp login`. func NewMcpLoginCmd() *cobra.Command { - return &cobra.Command{ + var auth *telemetryAuth + cmd := &cobra.Command{ Use: "login ", Short: "Sign in to a Databricks managed MCP service with your account", Long: `Sign in to a Databricks managed MCP service that uses your account with an external provider, such @@ -72,14 +69,18 @@ sign-in to complete. Without a browser, open the printed URL yourself. databricks aitools mcp login system.ai.github --profile logfood`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + auth.waitAtExit = true return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } + auth = withTelemetryAuth(cmd) + return cmd } // NewMcpLogoutCmd returns `aitools mcp logout`. func NewMcpLogoutCmd() *cobra.Command { - return &cobra.Command{ + var auth *telemetryAuth + cmd := &cobra.Command{ Use: "logout ", Short: "Remove your sign-in to a Databricks managed MCP service", Long: `Remove your sign-in to a Databricks managed MCP service on a profile. The install of the server @@ -90,9 +91,12 @@ stays in place. Running it when you are not signed in is not an error. databricks aitools mcp logout system.ai.github --profile logfood`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + auth.waitAtExit = true return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } + auth = withTelemetryAuth(cmd) + return cmd } // runMcpSignInCommand resolves the profile for service and runs action. In JSON mode the document is @@ -103,11 +107,8 @@ func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAct out := &mcpSignInOutput{Service: service} runErr := runMcpSignIn(ctx, cmd.ErrOrStderr(), profileFlag(cmd), service, action, out) - if out.cfg != nil { - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), out.cfg)) - } - // The event uploads with the profile's config, so a run that reached no server logs nothing. - if out.cfg != nil { + // The host is set once the profile's binding has passed and its client exists, so a run that reached no server logs nothing. + if out.Host != "" { logMcpEvent(ctx, signInEvent(operation, out, runErr)) } if !jsonMode { @@ -174,7 +175,6 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri return err } out.Host = w.Config.Host - out.cfg = w.Config return action(ctx, errOut, w, out, pinnedFile) } diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index b041a502b86..de22955aa5c 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -118,7 +119,8 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { } // TestMcpLoginRejectedTokenHasOneSetOfNextSteps checks the error that root prints for a rejected token after login has -// its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. +// its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. Root +// enriches the errors of a command whose upload config is set, which is the profile's config. func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { home := withHome(t) server := loginTestServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) @@ -127,8 +129,8 @@ func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.github"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogin, &out) require.Error(t, err) - require.NotNil(t, out.cfg, "the client exists, so root enriches errors") - enriched := auth.EnrichAuthError(t.Context(), out.cfg, err) + require.NotEmpty(t, out.Host, "the client exists, so the profile's config is known") + enriched := auth.EnrichAuthError(t.Context(), &config.Config{Host: out.Host, Profile: "logfood"}, err) assert.Equal(t, err.Error(), enriched.Error()) assert.Equal(t, 0, strings.Count(enriched.Error(), "Next steps:")) } From dd1c115dfc2c1b995b8d06db4eb11cf3cfb23336 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:40:35 +0000 Subject: [PATCH 14/43] Log PROXY_STARTED only for a serving start A start that fails its checks has no upload to make: a binding that failed has no profile the upload may authenticate with, and a rejected credential would only repeat its rejected request. serve logs its start once the checks have passed, and an event that names a rejected credential is not logged. Co-authored-by: Isaac --- .../telemetry/mcp-serve-rejected/output.txt | 13 +----- .../telemetry/mcp-serve-rejected/script | 2 +- .../telemetry/mcp-serve-rejected/test.toml | 4 +- .../aitools/telemetry/mcp-serve/output.txt | 2 +- .../cmd/aitools/telemetry/mcp-serve/script | 2 +- cmd/aitools/mcp_serve.go | 19 ++------- cmd/aitools/mcp_serve_test.go | 17 ++++++++ cmd/aitools/mcp_telemetry.go | 29 ++++++++------ cmd/aitools/mcp_telemetry_test.go | 40 +++++++++++++------ libs/telemetry/protos/aitools_mcp.go | 4 +- 10 files changed, 73 insertions(+), 59 deletions(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt index 18e5860dd73..89d332ab32f 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt @@ -1,18 +1,7 @@ -=== serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED +=== serve whose startup check is rejected: exit 2, and nothing is uploaded: a failed start logs no event >>> musterr [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] cannot read MCP service system.ai.genie_one_mcp: Invalid access token. Run: databricks auth login --profile logfood -{ - "calling_agent": "CLAUDE_CODE", - "operation": "PROXY_STARTED", - "servers": [ - { - "service": "GENIE_ONE", - "error_category": "AUTH_FAILED" - } - ], - "bound_profile_count": 0 -} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script index 25fb6292ef9..cf5022fbc50 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script @@ -3,6 +3,6 @@ envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg export BROWSER=none -title "serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED" +title "serve whose startup check is rejected: exit 2, and nothing is uploaded: a failed start logs no event" trace musterr $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < /dev/null > /dev/null mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml index c081d32cf7a..d60a4a6d774 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml @@ -1,5 +1,5 @@ -# The startup lookup of the service is rejected, so serve exits 2 before it serves. Its PROXY_STARTED event records the -# failure as AUTH_FAILED, and the upload that follows carries it. +# The startup lookup of the service is rejected, so serve exits 2 before it serves. A start that fails its checks logs no +# event, so no upload follows: the rejected token is not sent again. [[Server]] Pattern = "GET /api/2.1/unity-catalog/mcp-services/{name}" Response.StatusCode = 401 diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt index 1bcd45d41b1..d8fec5e3dcf 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -14,7 +14,7 @@ No install record for system.ai.genie_one_mcp on profile "logfood"; using its cu "bound_profile_count": 0 } -=== A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with +=== A binding that fails logs no PROXY_STARTED, so no upload follows >>> musterr [CLI] aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex MCP server system.ai.genie_one_mcp needs Databricks profile "gone", which does not exist in [TEST_TMP_DIR]/.databrickscfg. Set it up with: databricks auth login --profile gone diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script index 2e865c20274..0a5108cdb15 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -14,7 +14,7 @@ title "serve logs PROXY_STARTED after the binding checks; the upload is sent whe trace $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null mcp_events -title "A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with" +title "A binding that fails logs no PROXY_STARTED, so no upload follows" trace musterr $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null mcp_events diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 6eebff64ea2..45f82e15322 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -230,21 +230,9 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { Stderr: cmd.ErrOrStderr(), }) if err != nil { - // An AUTH_FAILED binding has passed the host check, so its profile may authenticate the upload. A profile whose - // host changed is not used: Bind never contacts a host the record does not name. - if errorCategory(err) == protos.AitoolsErrorCategoryAuthFailed { - if configFile, _, cerr := addConfigFile(cmd.Context()); cerr == nil { - setUploadConfig(cmd, profileName, configFile) - } - } - logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveBindFailure(cmd, profileName, err) } - // The bound profile authenticates the exit-time telemetry upload. The session's own config refreshes its token, which - // the upload must not do, so the upload gets a config of its own. - setUploadConfig(cmd, profileName, cfg.ConfigFile) - // A server pinned to a config file names that file in its hints, as add and login do. pinnedFile := mcp.PinnedFile(entry, cfg.ConfigFile) ctx, stop := serveSignalContext(cmd.Context()) @@ -268,7 +256,6 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. if refreshErr := refresh.failure(); refreshErr != nil { - cmd.SetContext(cmdctx.ClearConfigUsed(cmd.Context())) return serveFailure(cmd, refreshErr, serveBindStatus) } if ctx.Err() != nil { @@ -276,10 +263,12 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return nil } // serveChallenge fails only on an auth failure (see lookupAtStart), which exits serveBindStatus. - logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveFailure(cmd, err, serveBindStatus) } - logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, nil)) + // Only a serving start is logged. The upload authenticates with a config of its own that never refreshes a token + // (see setUploadConfig), since the session's config refreshes its token. + setUploadConfig(cmd, profileName, cfg.ConfigFile) + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service)) return serveProxy(ctx, cmd, refresh, proxy.Options{ In: cmd.InOrStdin(), diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index e3c3cbcd90e..25426c4a17e 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -30,7 +30,9 @@ import ( "github.com/databricks/cli/libs/auth/u2m" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/httpclient" @@ -558,6 +560,21 @@ func TestMcpServeRejectsServiceWithoutThreeParts(t *testing.T) { // TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it // exits 2 like the other binding failures. +// A start that fails its binding checks logs no PROXY_STARTED, so nothing uploads. +func TestMcpServeFailedStartLogsNothing(t *testing.T) { + withHome(t) + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := env.Set(telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + err := parent.ExecuteContext(ctx) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + assert.Equal(t, serveBindStatus, root.ExitStatus(err)) + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + assert.Empty(t, telemetry.BufferedEvents(serve.Context())) +} + func TestMcpServeStateReadFailureExitsTwo(t *testing.T) { withHome(t) dir, err := installer.GlobalSkillsDir(t.Context()) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index e3b623f8282..72266d9d7a4 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -2,6 +2,7 @@ package aitools import ( "context" + "slices" "strings" "github.com/databricks/cli/libs/aitools/agents" @@ -75,13 +76,20 @@ func proxyAgentType(flag string) protos.AitoolsAgentType { } // logMcpEvent buffers an MCP event; cmd/root uploads it at exit. An event without server results -// is not logged: a run that stopped before any server was reached has nothing to report. The bound -// profile count is read from the state file as it is after the operation. A state file that cannot -// be read drops the event; telemetry never fails a command. +// is not logged: a run that stopped before any server was reached has nothing to report. An event +// that names a rejected credential is not logged either: its upload would authenticate with that +// credential and be rejected the same way. The bound profile count is read from the state file as +// it is after the operation. A state file that cannot be read drops the event; telemetry never +// fails a command. func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) { if len(event.Servers) == 0 { return } + if slices.ContainsFunc(event.Servers, func(r protos.AitoolsMcpServerResult) bool { + return r.ErrorCategory == protos.AitoolsErrorCategoryAuthFailed + }) { + return + } state, err := mcp.LoadState(ctx) if err != nil { log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) @@ -215,18 +223,15 @@ func signOutServerResult(service, state string, err error) protos.AitoolsMcpServ return r } -// proxyStartedEvent is the PROXY_STARTED event of serve, logged once the startup checks have run. err is the failure -// of a check, or nil when the proxy is about to serve. The event uploads only when its profile resolved: a failed -// binding with no profile to upload with leaves it unsent. -func proxyStartedEvent(agent protos.AitoolsAgentType, service string, err error) protos.AitoolsMcpEvent { - r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} - if err != nil { - r.ErrorCategory = errorCategory(err) - } +// proxyStartedEvent is the PROXY_STARTED event of serve, logged once the startup checks have passed and the proxy serves. +// A start that fails its checks logs nothing. A binding that failed has no profile the upload may authenticate with +// (PROFILE_NOT_FOUND and PROFILE_HOST_CHANGED are never sent), and a rejected credential would only repeat its rejected +// request, so none of them uploads. +func proxyStartedEvent(agent protos.AitoolsAgentType, service string) protos.AitoolsMcpEvent { return protos.AitoolsMcpEvent{ CallingAgent: agent, Operation: protos.AitoolsMcpOperationProxyStarted, - Servers: []protos.AitoolsMcpServerResult{r}, + Servers: []protos.AitoolsMcpServerResult{{Service: mcpServiceType(service)}}, } } diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index b78201e3e29..8c2b8649ac0 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -9,6 +9,8 @@ import ( "testing" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -176,24 +178,36 @@ func TestSignOutServerResult(t *testing.T) { } func TestProxyStartedEvent(t *testing.T) { - ok := proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp", nil) assert.Equal(t, protos.AitoolsMcpEvent{ CallingAgent: protos.AitoolsAgentTypeClaudeCode, Operation: protos.AitoolsMcpOperationProxyStarted, Servers: []protos.AitoolsMcpServerResult{{Service: protos.AitoolsMcpServiceGenieOne}}, - }, ok) + }, proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp")) +} - // Each binding failure has its category; the error text never reaches the event. - for category, bind := range map[protos.AitoolsErrorCategory]mcp.BindCategory{ - protos.AitoolsErrorCategoryProfileNotFound: mcp.BindProfileNotFound, - protos.AitoolsErrorCategoryProfileHostChanged: mcp.BindProfileHostChanged, - protos.AitoolsErrorCategoryAuthFailed: mcp.BindAuthFailed, - } { - event := proxyStartedEvent(protos.AitoolsAgentTypeUnspecified, "main.tools.custom", &mcp.BindError{Category: bind}) - require.Len(t, event.Servers, 1) - assert.Equal(t, category, event.Servers[0].ErrorCategory) - assert.Equal(t, protos.AitoolsMcpServiceCustom, event.Servers[0].Service) - } +// An event that names a rejected credential is not logged: its upload would authenticate with that credential again. +func TestLogMcpEventDropsRejectedCredential(t *testing.T) { + withHome(t) + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryAuthFailed, + }}, + }) + assert.Empty(t, telemetry.BufferedEvents(ctx)) + + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryMCPPermissionDenied, + }}, + }) + assert.Len(t, telemetry.BufferedEvents(ctx), 1) } func TestSignInPromptedEvent(t *testing.T) { diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index f6a8d2f677b..0d7af69e138 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -64,8 +64,8 @@ const ( // AitoolsMcpServerResult is the result for one service, for one agent when the // operation writes agent config. Unset fields are omitted: an agent is not set // for sign-in, and an outcome is not set for proxy events. An error category is -// set when the target failed or was skipped, when a proxy start failed its -// checks (PROXY_STARTED), and on SIGN_IN_PROMPTED, where it is SIGN_IN_REQUIRED. +// set when the target failed or was skipped, and on SIGN_IN_PROMPTED, where it is +// SIGN_IN_REQUIRED. type AitoolsMcpServerResult struct { Service AitoolsMcpService `json:"service"` Agent AitoolsAgentType `json:"agent,omitempty"` From 9b947c75bf0527821cbbb6dea7a8b843bb1c8fbe Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:45:49 +0000 Subject: [PATCH 15/43] Tidy the MCP telemetry: sign-in and record-only states, service enum, docs An add whose sign-in failed before reading a state reports PENDING. A record-only removal that fails to save is reported FAILED. The service enum is one lookup that falls back to OTHER_SYSTEM. The server result doc says which outcomes carry an error category. Test comments name their test. Co-authored-by: Isaac --- cmd/aitools/mcp_add_test.go | 4 ++- cmd/aitools/mcp_login_test.go | 2 +- cmd/aitools/mcp_remove.go | 11 +++++++ cmd/aitools/mcp_remove_test.go | 37 ++++++++++++++++++++--- cmd/aitools/mcp_serve.go | 2 +- cmd/aitools/mcp_serve_test.go | 3 +- cmd/aitools/mcp_telemetry.go | 10 +++--- cmd/aitools/mcp_telemetry_test.go | 18 ++++++++--- libs/aitools/mcp/binding_test.go | 12 +++++++- libs/aitools/mcp/proxy/challenge_test.go | 8 +++-- libs/auth/error_test.go | 2 +- libs/telemetry/protos/aitools_mcp.go | 5 +-- libs/telemetry/protos/aitools_mcp_test.go | 7 +++-- 13 files changed, 93 insertions(+), 28 deletions(-) diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index 1ccd09a5e17..f503f0d0932 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -313,9 +313,11 @@ func withArg0(t *testing.T, arg0 string) { } // noRequestServer returns a server that fails the test on any request. A command that refuses before -// it contacts the profile's host never reaches it. +// it contacts the profile's host never reaches it. Telemetry is off: its auth resolution, which starts with the command, +// reads the host's metadata, a request the refusal does not make. func noRequestServer(t *testing.T) *httptest.Server { t.Helper() + t.Setenv("DATABRICKS_CLI_DISABLE_TELEMETRY", "1") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Errorf("unexpected request before the refusal: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusInternalServerError) diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index de22955aa5c..fb74488f556 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -120,7 +120,7 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { // TestMcpLoginRejectedTokenHasOneSetOfNextSteps checks the error that root prints for a rejected token after login has // its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. Root -// enriches the errors of a command whose upload config is set, which is the profile's config. +// enriches the errors of a command whose telemetry upload is authenticated with the profile. func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { home := withHome(t) server := loginTestServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 25cd0f640ca..726a5ba4937 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -261,6 +261,8 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso var drops []recordDrop var agentErrs []error + // recordOnly are the indexes of out.results whose removal only drops a state record. + var recordOnly []int for _, e := range selected { removed := removeRemoved{Name: e.Name, Service: e.Service, Profile: e.Profile, Agents: []string{}} pin := resolveBinding(ctx, e) @@ -280,6 +282,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // There is no config this CLI can edit for the agent, so only the record can go. removed.Agents = append(removed.Agents, agentName) removed.RecordOnly = append(removed.RecordOnly, agentName) + recordOnly = append(recordOnly, len(out.results)) out.results = append(out.results, removeServerResult(e.Service, agentName, removeDeleted, nil)) drops = append(drops, recordDrop{name: e.Name, agent: agentName}) continue @@ -301,6 +304,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // A server with no agent left in state is removed from state too, so it is reported as removed. Only its // record goes, which is a change to the state. if len(e.Agents) == 0 { + recordOnly = append(recordOnly, len(out.results)) out.results = append(out.results, protos.AitoolsMcpServerResult{Service: mcpServiceType(e.Service), Outcome: protos.AitoolsTargetOutcomeChanged}) } if len(removed.Agents) > 0 || len(e.Agents) == 0 { @@ -318,6 +322,13 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if len(drops) > 0 || slices.ContainsFunc(selected, func(e mcp.Entry) bool { return len(e.Agents) == 0 }) { saveErr = dropRecords(ctx, drops, selected) } + if saveErr != nil { + // A record-only removal changes nothing but the state, so the save that failed undid it. + for _, i := range recordOnly { + out.results[i].Outcome = protos.AitoolsTargetOutcomeFailed + out.results[i].ErrorCategory = errorCategory(saveErr) + } + } if !jsonMode { printRemoveResults(ctx, out) } diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index ddaf9d45f05..6a49d1c8cb8 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -1023,7 +1023,8 @@ func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel s return remove, err } -// A run that removes nothing logs no event, so it resolves no auth for an upload. +// TestMcpRemoveResolvesNothingWithoutAnEvent checks that a run that removes nothing logs no event, so it resolves no auth +// for an upload. func TestMcpRemoveResolvesNothingWithoutAnEvent(t *testing.T) { withHome(t) resolves := 0 @@ -1038,8 +1039,9 @@ func TestMcpRemoveResolvesNothingWithoutAnEvent(t *testing.T) { assert.Zero(t, resolves) } -// The upload is authenticated with the profile only when no server recorded for it has drifted: its host here is the one -// the server was set up against. The removal is done either way, and the output is written before the auth resolves. +// TestMcpRemoveUploadSkipsDriftedProfile checks that the upload is not authenticated with a profile that a server recorded +// for it no longer matches, as its host here is not the one the server was set up against. The removal is done either +// way, and the output is written before the auth resolves. func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = t\n") @@ -1064,7 +1066,7 @@ func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood", "the removal is done") } -// A profile whose recorded servers still match is used for the upload. +// TestMcpRemoveUploadUsesMatchingProfile checks that a profile whose recorded servers still match is used for the upload. func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") @@ -1084,7 +1086,8 @@ func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { assert.Equal(t, "https://adb-1.test", cmdctx.ConfigUsed(remove.Context()).Host) } -// An auth resolution that fails, as it does offline, leaves nothing in the output: its logs are debug only. +// TestMcpRemoveOfflineResolutionWarnsNothing checks that an auth resolution that fails, as it does offline, leaves nothing +// in the output: its logs are debug only. func TestMcpRemoveOfflineResolutionWarnsNothing(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") @@ -1103,3 +1106,27 @@ func TestMcpRemoveOfflineResolutionWarnsNothing(t *testing.T) { assert.Empty(t, logs.String()) assert.Contains(t, stdout.String(), `"removed"`) } + +// TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved checks that a removal that only drops state records is reported +// FAILED when the state cannot be saved, since nothing was removed. +func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { + withHome(t) + retired := mcp.Entry{ + Name: "dbsql-logfood", Service: "system.ai.dbsql", Profile: "logfood", Host: "https://adb-1.test", Command: testDatabricksCommand, + Agents: map[string]mcp.AgentRecord{"retired-agent": {Path: "/x/config.json", Scope: "user"}}, + } + saveState(t, retired) + saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } + t.Cleanup(func() { saveMCPState = mcp.SaveState }) + setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { return nil }) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "dbsql-logfood", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) + assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, events[0].AitoolsMcpEvent.Servers[0].ErrorCategory) +} diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 45f82e15322..098444c4cba 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -197,7 +197,7 @@ func NewMcpServeCmd() *cobra.Command { cmd.Flags().StringVar(&f.service, strings.TrimPrefix(mcpconfig.ServiceFlag, "--"), "", "Full name of the MCP service, for example system.ai.github") cmd.Flags().StringVar(&f.agent, strings.TrimPrefix(mcpconfig.AgentFlag, "--"), "", - "Registry name of the calling agent, used only for the User-Agent") + "Registry name of the calling agent, for the User-Agent and telemetry") return cmd } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index 25426c4a17e..6e4f327c71d 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -560,7 +560,8 @@ func TestMcpServeRejectsServiceWithoutThreeParts(t *testing.T) { // TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it // exits 2 like the other binding failures. -// A start that fails its binding checks logs no PROXY_STARTED, so nothing uploads. +// TestMcpServeFailedStartLogsNothing checks that a start that fails its binding checks logs no PROXY_STARTED, so nothing +// uploads. func TestMcpServeFailedStartLogsNothing(t *testing.T) { withHome(t) parent := newMCPTestRoot() diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 72266d9d7a4..1ccd734dacb 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -51,10 +51,10 @@ func mcpServiceType(fullName string) protos.AitoolsMcpService { if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], mcp.SystemAILocation) { return protos.AitoolsMcpServiceCustom } - if mcp.KnownOf(fullName) == nil { - return protos.AitoolsMcpServiceOtherSystem + if t, ok := mcpServiceTypes[strings.ToLower(parts[2])]; ok { + return t } - return mcpServiceTypes[strings.ToLower(parts[2])] + return protos.AitoolsMcpServiceOtherSystem } // callingAgentType returns the agent that drives the CLI, Unspecified when none is detected. @@ -114,9 +114,9 @@ func boundProfileCount(ctx context.Context, state *mcp.State) int64 { } // addSignInState is the sign-in state of a server after an add: the state that --login reports, else PENDING for a -// server that needs sign-in, else NOT_REQUIRED. +// server that needs sign-in, else NOT_REQUIRED. A sign-in that failed before it read a state is PENDING. func addSignInState(s addServerOutput) protos.AitoolsMcpSignInState { - if s.SignIn != nil { + if s.SignIn != nil && s.SignIn.State != "" { return signInStateOf(s.SignIn.State) } if s.SignInRequired { diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 8c2b8649ac0..deef70fd421 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -44,8 +44,8 @@ func TestMcpServiceType(t *testing.T) { } } -// Every service in the CLI's known table has its own enum, so a known service never reports OTHER_SYSTEM. A new table -// entry needs a new value in mcpServiceTypes. +// TestMcpServiceTypeCoversKnownServices checks that every service in the CLI's known table has its own enum, so a known +// service never reports OTHER_SYSTEM. A new table entry needs a new value in mcpServiceTypes. func TestMcpServiceTypeCoversKnownServices(t *testing.T) { for _, leaf := range mcp.KnownLeaves() { assert.Contains(t, mcpServiceTypes, leaf, "system.ai.%s has no telemetry enum: add it to mcpServiceTypes", leaf) @@ -185,7 +185,8 @@ func TestProxyStartedEvent(t *testing.T) { }, proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp")) } -// An event that names a rejected credential is not logged: its upload would authenticate with that credential again. +// TestLogMcpEventDropsRejectedCredential checks that an event that names a rejected credential is not logged: its upload +// would authenticate with that credential again. func TestLogMcpEventDropsRejectedCredential(t *testing.T) { withHome(t) ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") @@ -256,7 +257,8 @@ func TestAddSignInEvent(t *testing.T) { }, addSignInEvent("system.ai.github", signInStateActive, true, nil)) } -// The event carries enums and counts only. A custom service's name must not appear anywhere in the payload. +// TestMcpEventOmitsUserContent checks that the event carries enums and counts only: a custom service's name must not +// appear anywhere in the payload. func TestMcpEventOmitsUserContent(t *testing.T) { event := protos.AitoolsMcpEvent{ CallingAgent: protos.AitoolsAgentTypeClaudeCode, @@ -278,3 +280,11 @@ func TestMcpEventOmitsUserContent(t *testing.T) { assert.NotContains(t, string(b), "internal_tool") assert.Contains(t, string(b), `"service":"CUSTOM"`) } + +// TestAddSignInStateWithoutAState checks that a sign-in that failed before it read a state is PENDING, as a server that +// needs sign-in is until its state is known. A server that needs no sign-in is NOT_REQUIRED. +func TestAddSignInStateWithoutAState(t *testing.T) { + failed := addServerOutput{Service: "system.ai.github", SignInRequired: true, SignIn: &addSignInOutput{ErrorCategory: protos.AitoolsErrorCategoryUncategorized}} + assert.Equal(t, protos.AitoolsMcpSignInStatePending, addSignInState(failed)) + assert.Equal(t, protos.AitoolsMcpSignInStateNotRequired, addSignInState(addServerOutput{Service: "system.ai.dbsql"})) +} diff --git a/libs/aitools/mcp/binding_test.go b/libs/aitools/mcp/binding_test.go index f98905fa28b..d033c3c9024 100644 --- a/libs/aitools/mcp/binding_test.go +++ b/libs/aitools/mcp/binding_test.go @@ -947,7 +947,8 @@ func TestConfigFileCommandUsesForwardSlashes(t *testing.T) { mcp.ConfigFileCommand(path, "databricks auth login --profile logfood")) } -// The config of a telemetry upload never refreshes a token; the config a serve session binds keeps the default chain. +// TestBoundUploadConfigNeverRefreshes checks that the config of a telemetry upload never refreshes a token, and that the +// config a serve session binds keeps the default chain. func TestBoundUploadConfigNeverRefreshes(t *testing.T) { cfgPath := bindingTestConfig(t, bindingTestProfileContents(bindingTestToken)) @@ -959,3 +960,12 @@ func TestBoundUploadConfigNeverRefreshes(t *testing.T) { require.NoError(t, err) assert.NotEqual(t, auth.CredentialsWithoutRefresh(), session.Credentials) } + +// TestBindErrorNextStepsIncludedOnlyForAuthFailed checks that only an AUTH_FAILED message names the next steps, so root +// adds no second set of steps to the others. +func TestBindErrorNextStepsIncludedOnlyForAuthFailed(t *testing.T) { + assert.True(t, (&mcp.BindError{Category: mcp.BindAuthFailed}).NextStepsIncluded()) + for _, category := range []mcp.BindCategory{mcp.BindProfileNotFound, mcp.BindProfileHostChanged, mcp.BindUncategorized} { + assert.False(t, (&mcp.BindError{Category: category}).NextStepsIncluded(), category) + } +} diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 9abbc23ff59..822c625d661 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -273,8 +273,9 @@ func TestAuthenticateSaysAlreadySignedInAfterSignIn(t *testing.T) { require.NoError(t, p.finish(t)) } -// The prompt is recorded once per page that was opened, after its wait, with whether the credential -// became ACTIVE. A call that opens no page, or finds the caller already signed in, records nothing. +// TestAuthenticateRecordsPromptWhenPageOpens checks that the prompt is recorded once per page that was opened, after its +// wait, with whether the credential became ACTIVE. A call that opens no page, or finds the caller already signed in, +// records nothing. func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { @@ -304,7 +305,8 @@ func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { require.NoError(t, p.finish(t)) } -// A sign-in that fails before its URL is printed, such as a read of the workspace ID, shows no page: nothing is recorded. +// TestAuthenticateRecordsNoPromptWhenNoURLShown checks that a sign-in that fails before its URL is printed, such as a read +// of the workspace ID, shows no page, so nothing is recorded. func TestAuthenticateRecordsNoPromptWhenNoURLShown(t *testing.T) { srv, _ := challengedService(t) var calls int diff --git a/libs/auth/error_test.go b/libs/auth/error_test.go index b96a0386405..2ef1ff08ff5 100644 --- a/libs/auth/error_test.go +++ b/libs/auth/error_test.go @@ -81,7 +81,7 @@ func (e *stepsIncludedError) NextStepsIncluded() bool { } func TestEnrichAuthErrorLeavesErrorWithNextStepsAsIs(t *testing.T) { - cfg := &config.Config{Profile: "test", Host: "https://example.com", AuthType: "pat"} + cfg := &config.Config{Profile: "test", Host: "https://ws.test", AuthType: "pat"} original := &stepsIncludedError{cause: &apierr.APIError{StatusCode: 401, Message: "Unauthorized"}} assert.Equal(t, original, EnrichAuthError(t.Context(), cfg, original)) diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index 0d7af69e138..4e50bd67d46 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -64,8 +64,9 @@ const ( // AitoolsMcpServerResult is the result for one service, for one agent when the // operation writes agent config. Unset fields are omitted: an agent is not set // for sign-in, and an outcome is not set for proxy events. An error category is -// set when the target failed or was skipped, and on SIGN_IN_PROMPTED, where it is -// SIGN_IN_REQUIRED. +// set on a FAILED target and on SIGN_IN_PROMPTED, where it is SIGN_IN_REQUIRED. A +// SKIPPED target carries none: remove skips an entry it left in place, or one +// already gone, without a failure. type AitoolsMcpServerResult struct { Service AitoolsMcpService `json:"service"` Agent AitoolsAgentType `json:"agent,omitempty"` diff --git a/libs/telemetry/protos/aitools_mcp_test.go b/libs/telemetry/protos/aitools_mcp_test.go index 3b6ef512e3a..715a196103a 100644 --- a/libs/telemetry/protos/aitools_mcp_test.go +++ b/libs/telemetry/protos/aitools_mcp_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/require" ) -// The lumberjack proto keys these fields in snake_case. This pins the wire keys of an event -// and of one server result, and that unset enums are omitted rather than sent empty. +// TestAitoolsMcpEventEncodesSnakeCaseKeys checks that the lumberjack proto keys these fields in snake_case. It pins the +// wire keys of an event and of one server result, and that unset enums are omitted rather than sent empty. func TestAitoolsMcpEventEncodesSnakeCaseKeys(t *testing.T) { b, err := json.Marshal(DatabricksCliLog{AitoolsMcpEvent: &AitoolsMcpEvent{ CallingAgent: AitoolsAgentTypeClaudeCode, @@ -49,7 +49,8 @@ func TestAitoolsMcpEventEncodesSnakeCaseKeys(t *testing.T) { }, server) } -// A proxy event has no agent on the result, no outcome and no error category when it succeeded. +// TestAitoolsMcpServerResultOmitsUnsetFields checks that a proxy event has no agent on the result, no outcome and no error +// category when it succeeded. func TestAitoolsMcpServerResultOmitsUnsetFields(t *testing.T) { b, err := json.Marshal(AitoolsMcpServerResult{Service: AitoolsMcpServiceOtherSystem}) require.NoError(t, err) From 942533aad778e8c7a1b80a637802814fa0067175 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:13:14 +0000 Subject: [PATCH 16/43] Upload telemetry with a config the command sets The upload authenticates with a config a command sets once its binding checks have passed, in place of the one cmdctx.ConfigUsed holds, and a run can refuse its upload after a rejected credential. Commands that bind a profile use this instead of resolving the profile's auth in the background. Co-authored-by: Isaac --- libs/telemetry/logger.go | 45 +++++++++++++++++++++++-- libs/telemetry/logger_test.go | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index aac2063fbe5..97ffad8cd30 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -37,6 +37,39 @@ func Disabled(ctx context.Context) bool { return env.Get(ctx, disableEnvVar) != "" } +// SetUploadConfig sets the config that Upload authenticates with, in place of the one cmdctx.ConfigUsed holds. A command +// sets it once its own binding checks have passed, so that the upload authenticates with the profile the command bound. +// Like Log, it is a no-op without a logger. +func SetUploadConfig(ctx context.Context, cfg *config.Config) { + if l, ok := loggerFromContext(ctx); ok { + l.upload = cfg + } +} + +// RefuseUpload makes Upload send nothing for the run, whatever config is set. A run calls it once a credential was +// rejected, since the upload would present that credential again. Like Log, it is a no-op without a logger. +func RefuseUpload(ctx context.Context) { + if l, ok := loggerFromContext(ctx); ok { + l.refused = true + } +} + +// UploadConfig returns the config that Upload authenticates with: the one SetUploadConfig set, else the one +// cmdctx.ConfigUsed holds. It returns false when the run refused its upload or has no config. +func UploadConfig(ctx context.Context) (*config.Config, bool) { + l, ok := loggerFromContext(ctx) + if !ok || l.refused { + return nil, false + } + if l.upload != nil { + return l.upload, true + } + if cmdctx.HasConfigUsed(ctx) { + return cmdctx.ConfigUsed(ctx), true + } + return nil, false +} + func Log(ctx context.Context, event protos.DatabricksCliLog) { // A missing logger means telemetry was never initialized on this context // (e.g. a command invoked outside the normal cmd/root setup). Dropping the @@ -54,6 +87,11 @@ type logger struct { logs []protos.FrontendLog stateLineageHash string + // upload is the config Upload authenticates with, as SetUploadConfig set it. Nil leaves cmdctx.ConfigUsed to it. + upload *config.Config + // refused means the run refused its upload (RefuseUpload), so Upload sends nothing. + refused bool + // running counts the goroutines that Upload started and stopped waiting // for, which may still be using the config (see within). running atomic.Int32 @@ -123,8 +161,9 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { protoLogs[i] = string(b) } - if !cmdctx.HasConfigUsed(ctx) { - log.Debugf(ctx, "no auth config available; skipping telemetry upload") + cfg, ok := UploadConfig(ctx) + if !ok { + log.Debugf(ctx, "no auth config available, or the run refused its telemetry upload; skipping telemetry upload") return nil } @@ -150,7 +189,7 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { } apiClient, err := within(ctx, &l.running, func() (*client.DatabricksClient, error) { - return newAPIClient(cmdctx.ConfigUsed(ctx), &l.running) + return newAPIClient(cfg, &l.running) }) if err != nil { return fmt.Errorf("failed to create API client: %w", err) diff --git a/libs/telemetry/logger_test.go b/libs/telemetry/logger_test.go index 94673a9dd1a..a6a01c9a72e 100644 --- a/libs/telemetry/logger_test.go +++ b/libs/telemetry/logger_test.go @@ -465,3 +465,66 @@ func TestWithinReturnsPanicAsError(t *testing.T) { assert.EqualError(t, err, "panic: boom") assert.Eventually(t, func() bool { return running.Load() == 0 }, time.Second, time.Millisecond) } + +// TestUploadUsesConfigSetByCommand checks that the upload authenticates with the config the command set, in place of the +// one cmdctx.ConfigUsed holds. +func TestUploadUsesConfigSetByCommand(t *testing.T) { + bound := testserver.New(t) + t.Cleanup(bound.Close) + other := testserver.New(t) + t.Cleanup(other.Close) + var boundCalls, otherCalls int + bound.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + boundCalls++ + return ResponseBody{NumProtoSuccess: 1} + }) + other.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + otherCalls++ + return ResponseBody{NumProtoSuccess: 1} + }) + + ctx := WithNewLogger(t.Context()) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: other.URL, Token: "other"}) + SetUploadConfig(ctx, &config.Config{Host: bound.URL, Token: "bound"}) + + cfg, ok := UploadConfig(ctx) + require.True(t, ok) + assert.Equal(t, bound.URL, cfg.Host) + + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + assert.Equal(t, 1, boundCalls) + assert.Zero(t, otherCalls) +} + +// TestUploadRefusedSendsNothing checks that a run which refused its upload sends nothing, even with a config set by the +// command, and that a config set after the refusal is not used. +func TestUploadRefusedSendsNothing(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + var calls int + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + calls++ + return ResponseBody{NumProtoSuccess: 1} + }) + + ctx := WithNewLogger(t.Context()) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + SetUploadConfig(ctx, &config.Config{Host: server.URL, Token: "token"}) + RefuseUpload(ctx) + SetUploadConfig(ctx, &config.Config{Host: server.URL, Token: "token"}) + + _, ok := UploadConfig(ctx) + assert.False(t, ok) + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + assert.Zero(t, calls) +} + +// TestUploadConfigWithoutLogger checks that the upload slot is a no-op without a logger, as Log is. +func TestUploadConfigWithoutLogger(t *testing.T) { + ctx := t.Context() + SetUploadConfig(ctx, &config.Config{Host: "https://ws.test"}) + RefuseUpload(ctx) + _, ok := UploadConfig(ctx) + assert.False(t, ok) +} From 0cf806500cf22ad9bd9c016193807fe3f59d722e Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:19:23 +0000 Subject: [PATCH 17/43] Authenticate MCP telemetry uploads with the profile each command binds add, login, logout, remove and serve no longer resolve the upload's auth in the background, as install does. Each one sets the config of the profile it bound once its binding checks have passed, and an upload never refreshes a token. remove sets it only when the removed servers share one profile and none of the servers recorded for it has drifted. serve sets it at exit, after its refresh guard has run, and a rejected credential refuses the upload. Errors of these commands are no longer enriched by the root command, since their upload no longer sets the command's config. The auth error hint that skipped that enrichment, and cmdctx.ClearConfigUsed, are removed. A removal that prunes an entry is reported CHANGED, and a failed state save reports it FAILED. Co-authored-by: Isaac --- .../aitools/telemetry/mcp-remove/output.txt | 2 +- cmd/aitools/mcp_add.go | 8 +- cmd/aitools/mcp_add_test.go | 5 +- cmd/aitools/mcp_login.go | 7 +- cmd/aitools/mcp_login_test.go | 37 +++++--- cmd/aitools/mcp_remove.go | 43 +++++---- cmd/aitools/mcp_remove_test.go | 92 ++++++++++--------- cmd/aitools/mcp_serve.go | 14 +-- cmd/aitools/mcp_serve_test.go | 9 +- cmd/aitools/mcp_telemetry.go | 74 +++++++++------ cmd/aitools/mcp_telemetry_test.go | 4 +- libs/aitools/mcp/binding.go | 6 -- libs/aitools/mcp/binding_test.go | 9 -- libs/auth/error.go | 10 -- libs/auth/error_test.go | 27 ------ libs/cmdctx/config_used.go | 6 -- libs/cmdctx/config_used_test.go | 4 - 17 files changed, 164 insertions(+), 193 deletions(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt index 4f85fef980b..1d732f75094 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -73,7 +73,7 @@ { "service": "GENIE_ONE", "agent": "CLAUDE_CODE", - "outcome": "SKIPPED" + "outcome": "CHANGED" } ], "bound_profile_count": 0 diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index 91b35de3720..7d84fb133f3 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -160,7 +160,6 @@ func (o *addOutput) slashPaths() { // NewMcpAddCmd returns `aitools mcp add`. func NewMcpAddCmd() *cobra.Command { var f mcpAddFlags - var auth *telemetryAuth cmd := &cobra.Command{ Use: "add", Short: "Install Databricks managed MCP servers into coding agents", @@ -183,8 +182,6 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) - // The events upload with the auth resolved for the profile, and a run that logs one waits for it. - auth.waitAtExit = true out := &addOutput{ Scope: installer.ScopeGlobal, @@ -223,8 +220,6 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), cmd.Flags().BoolVar(&f.force, "force", false, "Overwrite an existing server with the same name that this CLI did not write") cmd.Flags().BoolVar(&f.login, "login", false, "Sign in to each sign-in-required server that was installed, after the installs") cmd.Flags().BoolVar(&f.noLogin, "no-login", false, "Do not sign in; print the login commands for sign-in-required servers (default without a terminal; in a terminal, the default asks for each one)") - // Resolve auth best-effort so the telemetry can upload, as install does. - auth = withTelemetryAuth(cmd) return cmd } @@ -371,6 +366,9 @@ func runMcpAdd(ctx context.Context, cmd *cobra.Command, f mcpAddFlags, jsonMode if err := planningRefusals(plans, targets, refused); err != nil { return err } + // Every refusal that needs no request has come, so the upload authenticates with the profile this run binds. Resolving + // its config makes the host's metadata request, which is why it waits for the refusals. + setUploadConfig(ctx, profileName, configFile) if !pickingServices { w, workspaceID, err = connectWorkspace(ctx, out, profileName, configFile, pinned) diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index f503f0d0932..210b5eec64d 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -313,11 +313,10 @@ func withArg0(t *testing.T, arg0 string) { } // noRequestServer returns a server that fails the test on any request. A command that refuses before -// it contacts the profile's host never reaches it. Telemetry is off: its auth resolution, which starts with the command, -// reads the host's metadata, a request the refusal does not make. +// it contacts the profile's host never reaches it, with telemetry on or off: the upload's config is resolved only after +// the binding checks have passed, and that resolution makes no request. func noRequestServer(t *testing.T) *httptest.Server { t.Helper() - t.Setenv("DATABRICKS_CLI_DISABLE_TELEMETRY", "1") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Errorf("unexpected request before the refusal: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusInternalServerError) diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index ed92d9b0657..7d865ffdcb0 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -56,7 +56,6 @@ type mcpSignInAction func(ctx context.Context, errOut io.Writer, w *databricks.W // NewMcpLoginCmd returns `aitools mcp login`. func NewMcpLoginCmd() *cobra.Command { - var auth *telemetryAuth cmd := &cobra.Command{ Use: "login ", Short: "Sign in to a Databricks managed MCP service with your account", @@ -69,17 +68,14 @@ sign-in to complete. Without a browser, open the printed URL yourself. databricks aitools mcp login system.ai.github --profile logfood`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - auth.waitAtExit = true return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } - auth = withTelemetryAuth(cmd) return cmd } // NewMcpLogoutCmd returns `aitools mcp logout`. func NewMcpLogoutCmd() *cobra.Command { - var auth *telemetryAuth cmd := &cobra.Command{ Use: "logout ", Short: "Remove your sign-in to a Databricks managed MCP service", @@ -91,11 +87,9 @@ stays in place. Running it when you are not signed in is not an error. databricks aitools mcp logout system.ai.github --profile logfood`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - auth.waitAtExit = true return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } - auth = withTelemetryAuth(cmd) return cmd } @@ -168,6 +162,7 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri // A mistyped --profile gets the existing profiles, as add lists them. return withExistingProfiles(ctx, state, profileName, err) } + setUploadConfig(ctx, profileName, configFile) // A server pinned to a config file names that file in the hints, as add does. pinnedFile := mcp.PinnedFile(entry, configFile) w, err := profileClient(ctx, profileName, configFile, pinnedFile) diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index fb74488f556..5ee7b9cf199 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -20,12 +20,12 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" - "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -118,21 +118,30 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { assert.Contains(t, err.Error(), "Run: databricks auth login --profile logfood") } -// TestMcpLoginRejectedTokenHasOneSetOfNextSteps checks the error that root prints for a rejected token after login has -// its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. Root -// enriches the errors of a command whose telemetry upload is authenticated with the profile. -func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { +// TestMcpLogoutForbiddenKeepsItsText checks that a 403 from logout is reported as the API said it, with no next steps +// added: root enriches a command's auth errors only when the command sets its profile's config for the context, and +// logout's upload config is not that config. The run still uploads with the profile it bound. +func TestMcpLogoutForbiddenKeepsItsText(t *testing.T) { home := withHome(t) - server := loginTestServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) - writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-expired\n") + server := loginTestServer(t, http.StatusForbidden, `{"error_code":"PERMISSION_DENIED","message":"no access"}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-logfood\n") + + parent := newMCPTestRoot() + parent.SetOut(&bytes.Buffer{}) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "logout", "system.ai.github", "--profile", "logfood"}) + ctx, _ := cmdio.NewTestContextWithStderr(telemetry.WithNewLogger(t.Context())) + err := parent.ExecuteContext(ctx) + logout, _, findErr := parent.Find([]string{"mcp", "logout"}) + require.NoError(t, findErr) - out := mcpSignInOutput{Service: "system.ai.github"} - err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogin, &out) require.Error(t, err) - require.NotEmpty(t, out.Host, "the client exists, so the profile's config is known") - enriched := auth.EnrichAuthError(t.Context(), &config.Config{Host: out.Host, Profile: "logfood"}, err) - assert.Equal(t, err.Error(), enriched.Error()) - assert.Equal(t, 0, strings.Count(enriched.Error(), "Next steps:")) + assert.Contains(t, err.Error(), "no access") + assert.NotContains(t, err.Error(), "Next steps") + assert.False(t, cmdctx.HasConfigUsed(logout.Context()), "root does not enrich the error") + cfg, uploads := telemetry.UploadConfig(logout.Context()) + require.True(t, uploads) + assert.Equal(t, "logfood", cfg.Profile) } // TestMcpLogoutUnknownServiceIsAnError checks that logout of a service the profile cannot see fails, rather than diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 726a5ba4937..ac8f832ef9e 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -12,9 +12,10 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/aitools/mcp/agentconfig" + "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -39,6 +40,9 @@ type removeOutput struct { results []protos.AitoolsMcpServerResult // stateBefore is the state as it was read before the removal. It is not in the JSON. stateBefore *mcp.State + // profile is the profile that the selected servers are bound to, when they all are bound to one. It is "" otherwise. + // It is not in the JSON. + profile string } type removeRemoved struct { @@ -114,7 +118,7 @@ removes one.`, if jsonMode { err = writeRemoveJSON(cmd, out, runErr) } - logRemoveEvent(cmd, args, out) + logRemoveEvent(ctx, out) return err }, } @@ -141,33 +145,31 @@ func writeRemoveJSON(cmd *cobra.Command, out *removeOutput, runErr error) error return nil } -// logRemoveEvent buffers the REMOVE event and sets the config its exit-time upload authenticates with. It runs after -// the output is written, and only when the run touched a server: a run that touched none has nothing to upload, so it -// resolves no auth. The auth resolves as install's does (startTelemetryAuth), within telemetryAuthBudget, and the upload -// is skipped when a server recorded for the profile no longer matches it (bindingDrifted). -func logRemoveEvent(cmd *cobra.Command, args []string, out *removeOutput) { - ctx := cmd.Context() - logMcpEvent(ctx, protos.AitoolsMcpEvent{ +// logRemoveEvent buffers the REMOVE event and sets the config its exit-time upload authenticates with. It runs after the +// output is written. A run that logged no event touched no server, so it has nothing to upload and sets no config. +func logRemoveEvent(ctx context.Context, out *removeOutput) { + if !logMcpEvent(ctx, protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), Operation: protos.AitoolsMcpOperationRemove, Servers: out.results, - }) - if len(telemetry.BufferedEvents(ctx)) == 0 { + }) { return } - auth := startTelemetryAuth(cmd, args) - if auth == nil { + if out.profile == "" { return } - if !auth.wait() { - log.Debugf(ctx, "auth for telemetry not resolved within %s; not uploading telemetry", telemetryAuthBudget) + configFile, err := mcp.ResolveConfigFile(ctx, env.Get(ctx, mcpconfig.ConfigFileEnv)) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve the config file to upload with: %s", err) return } - if auth.cfg != nil && bindingDrifted(ctx, out.stateBefore, auth.cfg) { - log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", auth.cfg.Profile) + // The upload sends the profile's token to the profile's host, so a profile whose recorded servers no longer match it + // uploads nothing. The records are the ones read before the removal. + if bindingDrifted(ctx, out.stateBefore, out.profile, configFile) { + log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", out.profile) return } - auth.apply(cmd, false) + setUploadConfig(ctx, out.profile, configFile) } // runMcpRemove removes the chosen servers from the agent configs and records the result. Results are printed @@ -244,6 +246,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // --agents chooses the servers installed for those agents, so a server with no agent left is not chosen. selected = slices.DeleteFunc(selected, func(e mcp.Entry) bool { return len(e.Agents) == 0 }) } + out.profile = sharedProfile(selected) // Each agent config file is read once per command, and each server's config file is resolved once. files := newAgentFiles() @@ -288,6 +291,10 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso continue } outcome, reason, err := removeFromAgent(ctx, files, e, a, pin, len(agentNames) > 0) + // A pruned entry changes the state only, so a save that fails undoes it, as for a record-only removal. + if outcome == removePruned && err == nil { + recordOnly = append(recordOnly, len(out.results)) + } out.results = append(out.results, removeServerResult(e.Service, agentName, outcome, err)) if err != nil { errs = append(errs, err) diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 6a49d1c8cb8..f9a95e0cb7e 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -11,19 +11,16 @@ import ( "path/filepath" "slices" "testing" - "time" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/log/handler" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" - "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1011,7 +1008,8 @@ func claudeServerEntry(claudePath, name, service string) mcp.Entry { func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel slog.Level, args ...string) (*cobra.Command, error) { t.Helper() parent := newMCPTestRoot() - ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + ctx := telemetry.WithNewLogger(t.Context()) + ctx = env.Set(ctx, "DATABRICKS_CLI_DISABLE_TELEMETRY", "") ctx = log.NewContext(ctx, slog.New(handler.NewFriendlyHandler(logs, &handler.Options{Level: logLevel}))) ctx, _ = cmdio.NewTestContextWithStderr(ctx) parent.SetOut(stdout) @@ -1023,25 +1021,27 @@ func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel s return remove, err } -// TestMcpRemoveResolvesNothingWithoutAnEvent checks that a run that removes nothing logs no event, so it resolves no auth -// for an upload. -func TestMcpRemoveResolvesNothingWithoutAnEvent(t *testing.T) { - withHome(t) - resolves := 0 - setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { - resolves++ - return nil - }) +// unsetConfigFile makes DATABRICKS_CONFIG_FILE unset for the test, as opposed to set to "". +func unsetConfigFile(t *testing.T) { + t.Setenv("DATABRICKS_CONFIG_FILE", "") + require.NoError(t, os.Unsetenv("DATABRICKS_CONFIG_FILE")) +} +// TestMcpRemoveSetsNoUploadWithoutAnEvent checks that a run that removes nothing logs no event, so it sets no config for +// an upload. +func TestMcpRemoveSetsNoUploadWithoutAnEvent(t *testing.T) { + withHome(t) var stdout, logs bytes.Buffer - _, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--all", "--output", "json") + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--all", "--output", "json") require.NoError(t, err) - assert.Zero(t, resolves) + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.False(t, uploads) + assert.Empty(t, telemetry.BufferedEvents(remove.Context())) } // TestMcpRemoveUploadSkipsDriftedProfile checks that the upload is not authenticated with a profile that a server recorded // for it no longer matches, as its host here is not the one the server was set up against. The removal is done either -// way, and the output is written before the auth resolves. +// way. func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = t\n") @@ -1052,57 +1052,65 @@ func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { saveState(t, entry) var stdout, logs bytes.Buffer - var outputWrittenFirst bool - setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { - outputWrittenFirst = stdout.Len() > 0 - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Profile: "logfood", Host: "https://adb-2.test", ConfigFile: filepath.Join(home, ".databrickscfg")})) - return nil - }) - remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") require.NoError(t, err) - assert.True(t, outputWrittenFirst, "the auth resolves after the JSON is written") - assert.False(t, cmdctx.HasConfigUsed(remove.Context()), "a drifted profile is not used to upload") + require.Len(t, telemetry.BufferedEvents(remove.Context()), 1, "the removal is logged") + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.False(t, uploads, "a drifted profile is not used to upload") assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood", "the removal is done") } -// TestMcpRemoveUploadUsesMatchingProfile checks that a profile whose recorded servers still match is used for the upload. +// TestMcpRemoveUploadUsesMatchingProfile checks that a profile whose recorded servers still match is used for the upload, +// read from the default config file when DATABRICKS_CONFIG_FILE is unset. func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { home := withHome(t) + unsetConfigFile(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") claude := agents.ByName("claude-code") entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") saveState(t, entry) - setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Profile: "logfood", Host: "https://adb-1.test", ConfigFile: filepath.Join(home, ".databrickscfg")})) - return nil - }) - var stdout, logs bytes.Buffer remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") require.NoError(t, err) - require.True(t, cmdctx.HasConfigUsed(remove.Context())) - assert.Equal(t, "https://adb-1.test", cmdctx.ConfigUsed(remove.Context()).Host) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "logfood", cfg.Profile) + assert.Equal(t, "https://adb-1.test", cfg.Host) + assert.Equal(t, "t", cfg.Token) } -// TestMcpRemoveOfflineResolutionWarnsNothing checks that an auth resolution that fails, as it does offline, leaves nothing -// in the output: its logs are debug only. -func TestMcpRemoveOfflineResolutionWarnsNothing(t *testing.T) { +// TestMcpRemoveUploadIgnoresEnvironmentToken checks that the upload authenticates with the profile's own token, never with +// a DATABRICKS_TOKEN of another workspace in the environment. +func TestMcpRemoveUploadIgnoresEnvironmentToken(t *testing.T) { home := withHome(t) + unsetConfigFile(t) + t.Setenv("DATABRICKS_TOKEN", "token-of-another-workspace") writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") claude := agents.ByName("claude-code") saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) - setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { - log.Warnf(cmd.Context(), "[hostmetadata] failed to fetch host metadata for https://adb-1.test") - return errors.New("cannot reach https://adb-1.test") - }) + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "t", cfg.Token) +} + +// TestMcpRemoveOfflineSetsUploadWithoutWarning checks that a remove that cannot reach the workspace warns nothing: the +// upload's config is resolved without a request, so there is no failure to report. +func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) var stdout, logs bytes.Buffer remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelWarn, "--names", "genie_one_mcp-logfood", "--output", "json") require.NoError(t, err) - assert.False(t, cmdctx.HasConfigUsed(remove.Context())) + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.True(t, uploads) assert.Empty(t, logs.String()) assert.Contains(t, stdout.String(), `"removed"`) } @@ -1118,7 +1126,6 @@ func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { saveState(t, retired) saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } t.Cleanup(func() { saveMCPState = mcp.SaveState }) - setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { return nil }) var stdout, logs bytes.Buffer remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "dbsql-logfood", "--output", "json") @@ -1126,6 +1133,7 @@ func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { events := telemetry.BufferedEvents(remove.Context()) require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, events[0].AitoolsMcpEvent.Servers[0].ErrorCategory) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 098444c4cba..d887b2af71c 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -26,10 +26,10 @@ import ( "github.com/databricks/cli/libs/auth/storage" "github.com/databricks/cli/libs/auth/u2m" "github.com/databricks/cli/libs/browser" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" @@ -265,12 +265,10 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { // serveChallenge fails only on an auth failure (see lookupAtStart), which exits serveBindStatus. return serveFailure(cmd, err, serveBindStatus) } - // Only a serving start is logged. The upload authenticates with a config of its own that never refreshes a token - // (see setUploadConfig), since the session's config refreshes its token. - setUploadConfig(cmd, profileName, cfg.ConfigFile) + // Only a serving start is logged. logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service)) - return serveProxy(ctx, cmd, refresh, proxy.Options{ + err = serveProxy(ctx, cmd, refresh, proxy.Options{ In: cmd.InOrStdin(), Out: cmd.OutOrStdout(), Err: cmd.ErrOrStderr(), @@ -285,6 +283,10 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { }, Challenge: challenge, }) + // The upload authenticates with the profile at exit, with a config of its own that never refreshes a token, since the + // session's config refreshes its token. A refresh failure has refused the upload (proxyResult), so nothing is set then. + setUploadConfig(cmd.Context(), profileName, cfg.ConfigFile) + return err } // bindingFor returns the state that the per-token binding check reads. With an install record, that is state. Without @@ -478,7 +480,7 @@ func proxyResult(cmd *cobra.Command, refresh *refreshGuard, runErr error) error if refreshErr := refresh.failure(); refreshErr != nil { // The token was rejected or the binding changed, so the upload at exit would only ask the token endpoint again or // use the re-pointed profile. - cmd.SetContext(cmdctx.ClearConfigUsed(cmd.Context())) + telemetry.RefuseUpload(cmd.Context()) fmt.Fprintf(cmd.ErrOrStderr(), "Error: stopped after %s for profile \"%s\"\n", stopReason(refreshErr), refresh.profile) return exitAlreadyPrinted(refreshErr, serveBindStatus) } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index 6e4f327c71d..45feda36e2f 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -28,7 +28,6 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth/storage" "github.com/databricks/cli/libs/auth/u2m" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" @@ -626,14 +625,16 @@ func TestProxyResultRefreshFailureWinsOverStdinError(t *testing.T) { } require.Error(t, g.authenticateRequest(&http.Request{Header: http.Header{}})) cmd := NewMcpServeCmd() - cmd.SetContext(cmdctx.SetConfigUsed(t.Context(), &config.Config{Host: "https://adb-1.test"})) + cmd.SetContext(telemetry.WithNewLogger(t.Context())) + telemetry.SetUploadConfig(cmd.Context(), &config.Config{Host: "https://adb-1.test"}) var stderr bytes.Buffer cmd.SetErr(&stderr) err := proxyResult(cmd, g, errors.New("read stdin: input/output error")) - // The exit-time upload would ask the rejected token's endpoint again, so it has no config. - assert.False(t, cmdctx.HasConfigUsed(cmd.Context())) + // The exit-time upload would ask the rejected token's endpoint again, so the run uploads nothing. + _, uploads := telemetry.UploadConfig(cmd.Context()) + assert.False(t, uploads) assert.ErrorIs(t, err, root.ErrAlreadyPrinted) assert.Equal(t, serveBindStatus, root.ExitStatus(err)) assert.ErrorIs(t, err, cause) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 1ccd734dacb..9e0492aed1e 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -7,25 +7,25 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" - "github.com/databricks/databricks-sdk-go/config" - "github.com/spf13/cobra" ) -// setUploadConfig sets the config of profileName in configFile for the exit-time upload, resolved within -// telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A profile that does not resolve -// uploads nothing. -func setUploadConfig(cmd *cobra.Command, profileName, configFile string) { - ctx := cmd.Context() +// setUploadConfig sets the config of profileName in configFile for the run's telemetry upload: resolved within +// telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A command calls it once its binding +// checks have passed, so the upload authenticates with the profile the command bound. Nothing is resolved when telemetry +// is disabled, and a profile that does not resolve uploads nothing. +func setUploadConfig(ctx context.Context, profileName, configFile string) { + if telemetry.Disabled(ctx) { + return + } cfg, err := mcp.BoundUploadConfig(profileName, configFile, telemetryAuthBudget) if err != nil { log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) return } - cmd.SetContext(cmdctx.SetConfigUsed(ctx, cfg)) + telemetry.SetUploadConfig(ctx, cfg) } // mcpServiceTypes maps the system.ai leaves in the CLI's service table to their telemetry enum. @@ -75,28 +75,29 @@ func proxyAgentType(flag string) protos.AitoolsAgentType { return agentType(flag) } -// logMcpEvent buffers an MCP event; cmd/root uploads it at exit. An event without server results -// is not logged: a run that stopped before any server was reached has nothing to report. An event -// that names a rejected credential is not logged either: its upload would authenticate with that -// credential and be rejected the same way. The bound profile count is read from the state file as -// it is after the operation. A state file that cannot be read drops the event; telemetry never -// fails a command. -func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) { +// logMcpEvent buffers an MCP event, and reports whether it did; cmd/root uploads it at exit. An event without server +// results is not logged: a run that stopped before any server was reached has nothing to report. An event that names a +// rejected credential is not logged either, and the run refuses its upload, since the upload would present that +// credential again. The bound profile count is read from the state file as it is after the operation. A state file that +// cannot be read drops the event; telemetry never fails a command. +func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) bool { if len(event.Servers) == 0 { - return + return false } if slices.ContainsFunc(event.Servers, func(r protos.AitoolsMcpServerResult) bool { return r.ErrorCategory == protos.AitoolsErrorCategoryAuthFailed }) { - return + telemetry.RefuseUpload(ctx) + return false } state, err := mcp.LoadState(ctx) if err != nil { log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) - return + return false } event.BoundProfileCount = boundProfileCount(ctx, state) telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsMcpEvent: &event}) + return true } // boundProfileCount returns the number of distinct profiles that have servers installed. A profile is its name in its @@ -151,18 +152,18 @@ func addServerResults(servers []addServerOutput) []protos.AitoolsMcpServerResult return results } -// removeServerResult returns the result of removing one server from one agent config. An entry that -// is gone already, or that was changed outside this CLI, is SKIPPED: nothing was changed there. +// removeServerResult returns the result of removing one server from one agent config. An entry that was deleted, or +// pruned (its record dropped), is CHANGED. An entry that was changed outside this CLI is SKIPPED: it is left in place. func removeServerResult(service, agent string, outcome removeOutcome, err error) protos.AitoolsMcpServerResult { r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), Agent: agentType(agent)} switch { case err != nil: r.Outcome = protos.AitoolsTargetOutcomeFailed r.ErrorCategory = errorCategory(err) - case outcome == removeDeleted: - r.Outcome = protos.AitoolsTargetOutcomeChanged - default: + case outcome == removeKept: r.Outcome = protos.AitoolsTargetOutcomeSkipped + default: + r.Outcome = protos.AitoolsTargetOutcomeChanged } return r } @@ -265,17 +266,30 @@ func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn } } -// bindingDrifted reports whether a server recorded for cfg's profile in state no longer matches the profile: its host or -// workspace changed, or its config file moved. The upload would send the profile's token to the host the profile names -// now, which the server was not set up against. It reads the config files only. -func bindingDrifted(ctx context.Context, state *mcp.State, cfg *config.Config) bool { +// bindingDrifted reports whether a server recorded for profileName in state no longer matches the profile in configFile: +// its host or workspace changed, or its config file moved. The upload would send the profile's token to the host the +// profile names now, which the server was not set up against. It reads the config files only. +func bindingDrifted(ctx context.Context, state *mcp.State, profileName, configFile string) bool { for _, e := range state.Servers { - if e.Profile != cfg.Profile { + if e.Profile != profileName { continue } - if _, err := mcp.CheckBinding(ctx, state, e.Service, cfg.Profile, cfg.ConfigFile); err != nil { + if _, err := mcp.CheckBinding(ctx, state, e.Service, profileName, configFile); err != nil { return true } } return false } + +// sharedProfile returns the profile that all entries are bound to, or "" when they are bound to none or to several. +func sharedProfile(entries []mcp.Entry) string { + if len(entries) == 0 { + return "" + } + for _, e := range entries[1:] { + if e.Profile != entries[0].Profile { + return "" + } + } + return entries[0].Profile +} diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index deef70fd421..742a7971348 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -104,8 +104,8 @@ func TestAddServerResults(t *testing.T) { func TestRemoveServerResult(t *testing.T) { assert.Equal(t, protos.AitoolsTargetOutcomeChanged, removeServerResult("system.ai.github", "claude-code", removeDeleted, nil).Outcome) - // An entry already gone, or changed outside this CLI, is left as it is. - assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, + // A pruned entry changes the state, so it is CHANGED too. An entry changed outside this CLI is left in place: SKIPPED. + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, removeServerResult("system.ai.github", "claude-code", removePruned, nil).Outcome) assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, removeServerResult("system.ai.github", "claude-code", removeKept, nil).Outcome) diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 79d473b01b8..6f2f83cc595 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -61,12 +61,6 @@ func (e *BindError) Unwrap() error { return e.cause } -// NextStepsIncluded reports whether the message already names the next steps: an AUTH_FAILED message names the login -// command for the profile. The root command does not add a second set of steps to it. -func (e *BindError) NextStepsIncluded() bool { - return e.Category == BindAuthFailed -} - // Bind checks that the profile still names the host and workspace recorded for the service, // then authenticates with the profile's own credentials. It never reads stdin, prompts, or // loads a bundle. It returns the resolved config and the matching state entry, which is nil diff --git a/libs/aitools/mcp/binding_test.go b/libs/aitools/mcp/binding_test.go index d033c3c9024..5b5f0f48af1 100644 --- a/libs/aitools/mcp/binding_test.go +++ b/libs/aitools/mcp/binding_test.go @@ -960,12 +960,3 @@ func TestBoundUploadConfigNeverRefreshes(t *testing.T) { require.NoError(t, err) assert.NotEqual(t, auth.CredentialsWithoutRefresh(), session.Credentials) } - -// TestBindErrorNextStepsIncludedOnlyForAuthFailed checks that only an AUTH_FAILED message names the next steps, so root -// adds no second set of steps to the others. -func TestBindErrorNextStepsIncludedOnlyForAuthFailed(t *testing.T) { - assert.True(t, (&mcp.BindError{Category: mcp.BindAuthFailed}).NextStepsIncluded()) - for _, category := range []mcp.BindCategory{mcp.BindProfileNotFound, mcp.BindProfileHostChanged, mcp.BindUncategorized} { - assert.False(t, (&mcp.BindError{Category: category}).NextStepsIncluded(), category) - } -} diff --git a/libs/auth/error.go b/libs/auth/error.go index 5e5b6369c24..fdb5f256f43 100644 --- a/libs/auth/error.go +++ b/libs/auth/error.go @@ -82,13 +82,6 @@ func RewriteAuthError(ctx context.Context, host, accountId, profile string, err return false, err } -// nextStepsIncluded is implemented by an error whose message already gives the next steps for the failure, such as the -// login command of an MCP server. EnrichAuthError leaves such an error as it is, so the user is given one set of steps. -type nextStepsIncluded interface { - error - NextStepsIncluded() bool -} - // EnrichAuthError appends identity context and remediation steps to 401/403 API errors. // For non-API errors or other status codes, the original error is returned unchanged. func EnrichAuthError(ctx context.Context, cfg *config.Config, err error) error { @@ -99,9 +92,6 @@ func EnrichAuthError(ctx context.Context, cfg *config.Config, err error) error { if apiErr.StatusCode != http.StatusUnauthorized && apiErr.StatusCode != http.StatusForbidden { return err } - if steps, ok := errors.AsType[nextStepsIncluded](err); ok && steps.NextStepsIncluded() { - return err - } var b strings.Builder diff --git a/libs/auth/error_test.go b/libs/auth/error_test.go index 2ef1ff08ff5..ce8f0df1b63 100644 --- a/libs/auth/error_test.go +++ b/libs/auth/error_test.go @@ -63,33 +63,6 @@ func TestEnrichAuthError_PreservesOriginalError(t *testing.T) { assert.Equal(t, "PERMISSION_DENIED", unwrapped.ErrorCode) } -// stepsIncludedError is an authentication failure whose message already gives the next steps. -type stepsIncludedError struct { - cause error -} - -func (e *stepsIncludedError) Error() string { - return "credential rejected. Run: databricks auth login --profile test" -} - -func (e *stepsIncludedError) Unwrap() error { - return e.cause -} - -func (e *stepsIncludedError) NextStepsIncluded() bool { - return true -} - -func TestEnrichAuthErrorLeavesErrorWithNextStepsAsIs(t *testing.T) { - cfg := &config.Config{Profile: "test", Host: "https://ws.test", AuthType: "pat"} - original := &stepsIncludedError{cause: &apierr.APIError{StatusCode: 401, Message: "Unauthorized"}} - assert.Equal(t, original, EnrichAuthError(t.Context(), cfg, original)) - - // An API error without a message of its own is still enriched. - plain := &apierr.APIError{StatusCode: 401, Message: "Unauthorized"} - assert.Contains(t, EnrichAuthError(t.Context(), cfg, plain).Error(), "Next steps:") -} - func TestEnrichAuthError(t *testing.T) { tests := []struct { name string diff --git a/libs/cmdctx/config_used.go b/libs/cmdctx/config_used.go index ed20650aea1..40595ef8aa9 100644 --- a/libs/cmdctx/config_used.go +++ b/libs/cmdctx/config_used.go @@ -18,12 +18,6 @@ func ConfigUsed(ctx context.Context) *config.Config { return cfg } -// ClearConfigUsed returns a context with no config used, as if SetConfigUsed had not run: the telemetry upload is -// skipped and auth errors are not enriched. -func ClearConfigUsed(ctx context.Context) context.Context { - return context.WithValue(ctx, configUsedKey, nil) -} - func HasConfigUsed(ctx context.Context) bool { _, ok := ctx.Value(configUsedKey).(*config.Config) return ok diff --git a/libs/cmdctx/config_used_test.go b/libs/cmdctx/config_used_test.go index 8a0724ceb76..f99febf8046 100644 --- a/libs/cmdctx/config_used_test.go +++ b/libs/cmdctx/config_used_test.go @@ -27,10 +27,6 @@ func TestCommandConfigUsed(t *testing.T) { // The config should have the correct configuration. assert.Equal(t, "https://test.test", ConfigUsed(ctx).Host) - // Clearing the config leaves none, as if it had never been set. - assert.False(t, HasConfigUsed(ClearConfigUsed(ctx))) - assert.True(t, HasConfigUsed(ctx)) - // Second call should update the config used. cfg2 := &config.Config{ Host: "https://test2.test", From a5aafa2b487d3fc4ce777691e9ee6ba991275e6f Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:21:25 +0000 Subject: [PATCH 18/43] Record sign-in prompts of waits on a cancelled page, and test the refusal and removal outcomes An authenticate call that waits on the page an earlier cancelled call showed records its outcome as a prompt too, so a sign-in that completes there is ACTIVE rather than PENDING. The proto mirror says a retry reports its own outcome. Tests cover the upload refusal after a rejected credential, a pruned entry reported CHANGED (FAILED when the state is not saved), and a failed state read after another event. Co-authored-by: Isaac --- cmd/aitools/mcp_remove_test.go | 58 ++++++++++++++++++++++++ cmd/aitools/mcp_telemetry_test.go | 27 +++++++---- libs/aitools/mcp/proxy/challenge.go | 7 +-- libs/aitools/mcp/proxy/challenge_test.go | 52 +++++++++++++++++++-- libs/telemetry/protos/aitools_mcp.go | 12 +++-- 5 files changed, 135 insertions(+), 21 deletions(-) diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index f9a95e0cb7e..f2f82b84ed9 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" @@ -1115,6 +1116,63 @@ func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { assert.Contains(t, stdout.String(), `"removed"`) } +// TestMcpRemovePrunedEntryReportsChanged checks that an entry the agent config no longer holds is reported CHANGED, since +// its record is dropped, and FAILED when the state cannot be saved, as a record-only removal is. +func TestMcpRemovePrunedEntryReportsChanged(t *testing.T) { + home := withHome(t) + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + seedPruned := func() { + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + require.NoError(t, os.WriteFile(claudePath, []byte(`{"mcpServers":{}}`), 0o600)) + } + + seedPruned() + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, events[0].AitoolsMcpEvent.Servers[0].Outcome) + + seedPruned() + saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } + t.Cleanup(func() { saveMCPState = mcp.SaveState }) + stdout.Reset() + remove, err = runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + events = telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) +} + +// TestMcpRemoveFailedStateReadAfterAnotherEventLogsNothing checks that a remove that cannot read the state logs nothing +// of its own, even when the run already buffered another event, and does not read the state it never read. +func TestMcpRemoveFailedStateReadAfterAnotherEventLogsNothing(t *testing.T) { + withHome(t) + dir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".state.json"), 0o700)) + + parent := newMCPTestRoot() + parent.SetOut(&bytes.Buffer{}) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "remove", "--all", "--output", "json"}) + ctx := telemetry.WithNewLogger(t.Context()) + telemetry.Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx, _ = cmdio.NewTestContextWithStderr(ctx) + require.Error(t, parent.ExecuteContext(ctx)) + remove, _, findErr := parent.Find([]string{"mcp", "remove"}) + require.NoError(t, findErr) + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + assert.NotNil(t, events[0].CliTestEvent) +} + // TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved checks that a removal that only drops state records is reported // FAILED when the state cannot be saved, since nothing was removed. func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 742a7971348..df29d6c9237 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -185,30 +186,36 @@ func TestProxyStartedEvent(t *testing.T) { }, proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp")) } -// TestLogMcpEventDropsRejectedCredential checks that an event that names a rejected credential is not logged: its upload -// would authenticate with that credential again. +// TestLogMcpEventDropsRejectedCredential checks that an event that names a rejected credential is not logged, and that the +// run then uploads nothing: its upload would authenticate with that credential again. The events of the run that were +// logged before stay buffered, and a later event of the run is logged, but not uploaded. func TestLogMcpEventDropsRejectedCredential(t *testing.T) { withHome(t) ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") - logMcpEvent(ctx, protos.AitoolsMcpEvent{ + telemetry.SetUploadConfig(ctx, &config.Config{Profile: "logfood", Host: "https://adb-1.test"}) + assert.False(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ Operation: protos.AitoolsMcpOperationSignIn, Servers: []protos.AitoolsMcpServerResult{{ Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, ErrorCategory: protos.AitoolsErrorCategoryAuthFailed, }}, - }) + })) assert.Empty(t, telemetry.BufferedEvents(ctx)) + _, uploads := telemetry.UploadConfig(ctx) + assert.False(t, uploads) - logMcpEvent(ctx, protos.AitoolsMcpEvent{ - Operation: protos.AitoolsMcpOperationSignIn, + assert.True(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ + Operation: protos.AitoolsMcpOperationAdd, Servers: []protos.AitoolsMcpServerResult{{ - Service: protos.AitoolsMcpServiceGitHub, - Outcome: protos.AitoolsTargetOutcomeFailed, - ErrorCategory: protos.AitoolsErrorCategoryMCPPermissionDenied, + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStatePending, }}, - }) + })) assert.Len(t, telemetry.BufferedEvents(ctx), 1) + _, uploads = telemetry.UploadConfig(ctx) + assert.False(t, uploads, "the add event is logged, and it is not uploaded with the rejected run") } func TestSignInPromptedEvent(t *testing.T) { diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index 1dbcffb0391..a5ff5a77dea 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -53,8 +53,9 @@ type SignInFunc func(ctx context.Context, req SignInRequest) (alreadySignedIn, u // Recorder receives the proxy's telemetry events. The command implements it, so the proxy does not // build telemetry events itself. type Recorder interface { - // SignInPrompted runs after the wait of an authenticate call that opened the sign-in page. - // signedIn reports whether the credential became ACTIVE. + // SignInPrompted runs after the wait of an authenticate call that showed the sign-in page: one it opened, or one an + // earlier cancelled call opened, which the user may still have open. signedIn reports whether the credential became + // ACTIVE. A retry after a cancelled call reports its own outcome. SignInPrompted(ctx context.Context, signedIn bool) } @@ -519,7 +520,7 @@ func (h *signInHooks) authenticate(ctx context.Context, id, progressToken json.R }) // A prompt is a page the user was shown. A caller that was already signed in, or a sign-in that failed before its // URL was printed, prompted nobody. - if open && !alreadySignedIn && urlShown { + if !alreadySignedIn && urlShown { h.c.Recorder.SignInPrompted(ctx, err == nil) } h.pageCancelled = urlShown && err != nil && ctx.Err() != nil diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 822c625d661..b7a46e0f550 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -273,9 +273,8 @@ func TestAuthenticateSaysAlreadySignedInAfterSignIn(t *testing.T) { require.NoError(t, p.finish(t)) } -// TestAuthenticateRecordsPromptWhenPageOpens checks that the prompt is recorded once per page that was opened, after its -// wait, with whether the credential became ACTIVE. A call that opens no page, or finds the caller already signed in, -// records nothing. +// TestAuthenticateRecordsPromptWhenPageOpens checks that the prompt is recorded once per page that was shown, after its +// wait, with whether the credential became ACTIVE. A call that finds the caller already signed in records nothing. func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { @@ -330,6 +329,53 @@ func TestAuthenticateRecordsNoPromptWhenNoURLShown(t *testing.T) { require.NoError(t, p.finish(t)) } +// TestAuthenticateRecordsPromptOfCallThatWaitsOnCancelledPage checks that a call which waits on the page an earlier, +// cancelled call showed records its own outcome: the page was shown for it too, so a sign-in that completes there is +// recorded as ACTIVE, not left PENDING from the cancelled call. +func TestAuthenticateRecordsPromptOfCallThatWaitsOnCancelledPage(t *testing.T) { + srv, _ := newFakeEndpoint(t, answerWith(http.StatusOK, listChallenge)) + entered, done := make(chan struct{}), make(chan struct{}) + var mu sync.Mutex + var opens []bool + signIn := func(ctx context.Context, req SignInRequest) (bool, bool, error) { + mu.Lock() + opens = append(opens, req.Open) + first := len(opens) == 1 + mu.Unlock() + if !first { + return false, true, nil + } + close(entered) + <-ctx.Done() + close(done) + return false, true, ctx.Err() + } + recorder := &promptRecorder{} + c := testChallenge(signIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + authenticate := `{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}` + + p.send(t, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`) + p.waitLines(t, 1) + p.send(t, fmt.Sprintf(authenticate, 5)) + waitClosed(t, entered, "the first sign-in to start") + p.send(t, `{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":5}}`) + waitClosed(t, done, "the first sign-in to return") + p.send(t, fmt.Sprintf(authenticate, 6)) + require.Eventually(t, func() bool { + return len(recorder.prompts()) == 2 + }, 10*time.Second, time.Millisecond) + require.NoError(t, p.finish(t)) + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, []bool{true, false}, opens, "the second call does not open the page again") + assert.Equal(t, []bool{false, true}, recorder.prompts(), "the cancelled call is PENDING, and the second call is ACTIVE") +} + func TestAuthenticateReopensPageAfterTimeout(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index 4e50bd67d46..5546025d526 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -9,11 +9,13 @@ package protos type AitoolsMcpOperation string const ( - AitoolsMcpOperationAdd AitoolsMcpOperation = "ADD" - AitoolsMcpOperationRemove AitoolsMcpOperation = "REMOVE" - AitoolsMcpOperationSignIn AitoolsMcpOperation = "SIGN_IN" - AitoolsMcpOperationSignOut AitoolsMcpOperation = "SIGN_OUT" - AitoolsMcpOperationProxyStarted AitoolsMcpOperation = "PROXY_STARTED" + AitoolsMcpOperationAdd AitoolsMcpOperation = "ADD" + AitoolsMcpOperationRemove AitoolsMcpOperation = "REMOVE" + AitoolsMcpOperationSignIn AitoolsMcpOperation = "SIGN_IN" + AitoolsMcpOperationSignOut AitoolsMcpOperation = "SIGN_OUT" + AitoolsMcpOperationProxyStarted AitoolsMcpOperation = "PROXY_STARTED" + // AitoolsMcpOperationSignInPrompted is logged for each authenticate call that showed the sign-in page, with the + // state of the sign-in when that call's wait ended. A retry after a cancelled call logs its own event. AitoolsMcpOperationSignInPrompted AitoolsMcpOperation = "SIGN_IN_PROMPTED" ) From 7e155fe82c190974b18a56ae0563d4753f00a7d6 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:23:43 +0000 Subject: [PATCH 19/43] Tidy the MCP telemetry: one SIGN_IN builder, rejected-start test, documented enum mirrors signInEventOf is the one SIGN_IN event builder, used by add --login and login. A misplaced test doc comment moves to its test, and remove's doc says the edit needs no network. The rejected-start case of the telemetry acceptance tests is replaced by a unit test that a rejected startup read logs nothing. KnownLeaves has its own test, and the enum values that no code names have comments that they mirror the schema. Co-authored-by: Isaac --- .../mcp-serve-rejected/.databrickscfg | 3 --- .../mcp-serve-rejected/out.test.toml | 1 - .../telemetry/mcp-serve-rejected/output.txt | 7 ------ .../telemetry/mcp-serve-rejected/script | 8 ------ .../telemetry/mcp-serve-rejected/test.toml | 6 ----- cmd/aitools/mcp_add.go | 4 +-- cmd/aitools/mcp_login.go | 7 ++---- cmd/aitools/mcp_remove.go | 2 +- cmd/aitools/mcp_serve_test.go | 25 +++++++++++++++++-- cmd/aitools/mcp_telemetry.go | 4 +-- cmd/aitools/mcp_telemetry_test.go | 4 +-- libs/aitools/mcp/known_test.go | 13 ++++++++++ libs/telemetry/protos/aitools_install.go | 6 +++-- libs/telemetry/protos/aitools_mcp.go | 5 ++-- 14 files changed, 52 insertions(+), 43 deletions(-) delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg deleted file mode 100644 index 0369be633c5..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg +++ /dev/null @@ -1,3 +0,0 @@ -[logfood] -host = $DATABRICKS_HOST -token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml deleted file mode 100644 index 18b1a88417e..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml +++ /dev/null @@ -1 +0,0 @@ -Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt deleted file mode 100644 index 89d332ab32f..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt +++ /dev/null @@ -1,7 +0,0 @@ - -=== serve whose startup check is rejected: exit 2, and nothing is uploaded: a failed start logs no event - ->>> musterr [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code -No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] -cannot read MCP service system.ai.genie_one_mcp: Invalid access token. -Run: databricks auth login --profile logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script deleted file mode 100644 index cf5022fbc50..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script +++ /dev/null @@ -1,8 +0,0 @@ -sethome home -envsubst < .databrickscfg > out && mv out .databrickscfg -export DATABRICKS_CONFIG_FILE=.databrickscfg -export BROWSER=none - -title "serve whose startup check is rejected: exit 2, and nothing is uploaded: a failed start logs no event" -trace musterr $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < /dev/null > /dev/null -mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml deleted file mode 100644 index d60a4a6d774..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# The startup lookup of the service is rejected, so serve exits 2 before it serves. A start that fails its checks logs no -# event, so no upload follows: the rejected token is not sent again. -[[Server]] -Pattern = "GET /api/2.1/unity-catalog/mcp-services/{name}" -Response.StatusCode = 401 -Response.Body = '{"error_code": "UNAUTHENTICATED", "message": "Invalid access token."}' diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index 7d84fb133f3..4ae3d902930 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -1258,7 +1258,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp if signedIn(ctx, w, plans[i].service.FullName) { server.SignIn = &addSignInOutput{State: signInStateActive} reportSignedIn(errOut, plans[i].service, req.profile, true) - logMcpEvent(ctx, addSignInEvent(server.Service, signInStateActive, true, nil)) + logMcpEvent(ctx, signInEventOf(server.Service, signInStateActive, true, nil)) continue } yes, err := promptSignInNow(server.Service) @@ -1279,7 +1279,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp state, alreadySignedIn, _, err = signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) } server.SignIn = &addSignInOutput{State: state} - logMcpEvent(ctx, addSignInEvent(server.Service, state, alreadySignedIn, err)) + logMcpEvent(ctx, signInEventOf(server.Service, state, alreadySignedIn, err)) if err != nil { server.SignIn.ErrorCategory = errorCategory(err) server.SignIn.Error = err.Error() diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index 7d865ffdcb0..0423e9db8fd 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -175,16 +175,13 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri // signInEvent is the SIGN_IN or SIGN_OUT event of one login or logout. func signInEvent(operation protos.AitoolsMcpOperation, out *mcpSignInOutput, err error) protos.AitoolsMcpEvent { - var r protos.AitoolsMcpServerResult if operation == protos.AitoolsMcpOperationSignIn { - r = signInServerResult(out.Service, out.State, out.alreadySignedIn, err) - } else { - r = signOutServerResult(out.Service, out.State, err) + return signInEventOf(out.Service, out.State, out.alreadySignedIn, err) } return protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), Operation: operation, - Servers: []protos.AitoolsMcpServerResult{r}, + Servers: []protos.AitoolsMcpServerResult{signOutServerResult(out.Service, out.State, err)}, } } diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index ac8f832ef9e..ec344724c02 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -83,7 +83,7 @@ type recordDrop struct { agent string } -// NewMcpRemoveCmd returns `aitools mcp remove`. It needs no network. +// NewMcpRemoveCmd returns `aitools mcp remove`. The edit needs no network; the telemetry upload after it is best-effort. func NewMcpRemoveCmd() *cobra.Command { var f mcpRemoveFlags cmd := &cobra.Command{ diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index 45feda36e2f..b4db7b31d46 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -557,8 +557,6 @@ func TestMcpServeRejectsServiceWithoutThreeParts(t *testing.T) { } } -// TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it -// exits 2 like the other binding failures. // TestMcpServeFailedStartLogsNothing checks that a start that fails its binding checks logs no PROXY_STARTED, so nothing // uploads. func TestMcpServeFailedStartLogsNothing(t *testing.T) { @@ -575,6 +573,29 @@ func TestMcpServeFailedStartLogsNothing(t *testing.T) { assert.Empty(t, telemetry.BufferedEvents(serve.Context())) } +// TestMcpServeRejectedStartLogsNothing checks that a start whose startup read is rejected by the profile's credential logs +// no PROXY_STARTED, so the run uploads nothing: the rejected token is not sent again. +func TestMcpServeRejectedStartLogsNothing(t *testing.T) { + home := withHome(t) + server := serviceServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-expired\n") + + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + err := parent.ExecuteContext(ctx) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + assert.Equal(t, serveBindStatus, root.ExitStatus(err)) + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + assert.Empty(t, telemetry.BufferedEvents(serve.Context())) + _, uploads := telemetry.UploadConfig(serve.Context()) + assert.False(t, uploads) +} + +// TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it +// exits 2 like the other binding failures. func TestMcpServeStateReadFailureExitsTwo(t *testing.T) { withHome(t) dir, err := installer.GlobalSkillsDir(t.Context()) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 9e0492aed1e..ca94f89e33c 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -198,9 +198,9 @@ func signInServerResult(service, state string, alreadySignedIn bool, err error) return r } -// addSignInEvent is the SIGN_IN event of one provider sign-in under add --login. A sign-in that found the caller +// signInEventOf is the SIGN_IN event of one provider sign-in, under add --login or login. A sign-in that found the caller // already signed in is UNCHANGED. -func addSignInEvent(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpEvent { +func signInEventOf(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpEvent { return protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), Operation: protos.AitoolsMcpOperationSignIn, diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index df29d6c9237..ccd28d3e870 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -251,7 +251,7 @@ func TestBoundProfileCount(t *testing.T) { assert.Equal(t, int64(0), boundProfileCount(t.Context(), &mcp.State{})) } -func TestAddSignInEvent(t *testing.T) { +func TestSignInEventOf(t *testing.T) { // An ACTIVE credential that the add found signed in is UNCHANGED, as add --login reports it. assert.Equal(t, protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), @@ -261,7 +261,7 @@ func TestAddSignInEvent(t *testing.T) { Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateActive, }}, - }, addSignInEvent("system.ai.github", signInStateActive, true, nil)) + }, signInEventOf("system.ai.github", signInStateActive, true, nil)) } // TestMcpEventOmitsUserContent checks that the event carries enums and counts only: a custom service's name must not diff --git a/libs/aitools/mcp/known_test.go b/libs/aitools/mcp/known_test.go index 9f8face3bc0..33b13ad0183 100644 --- a/libs/aitools/mcp/known_test.go +++ b/libs/aitools/mcp/known_test.go @@ -1,6 +1,7 @@ package mcp import ( + "slices" "testing" "github.com/stretchr/testify/assert" @@ -40,6 +41,18 @@ func TestKnownSystemAITable(t *testing.T) { } } +// TestKnownLeavesAreTheSystemAITable checks that KnownLeaves lists the leaves of the system.ai table, sorted, with each +// leaf known as a system.ai service. +func TestKnownLeavesAreTheSystemAITable(t *testing.T) { + leaves := KnownLeaves() + assert.Contains(t, leaves, "genie_one_mcp") + assert.True(t, slices.IsSorted(leaves)) + assert.Len(t, leaves, len(knownSystemAI)) + for _, leaf := range leaves { + assert.NotNil(t, KnownOf(SystemAILocation+"."+leaf), leaf) + } +} + func TestProviderLabel(t *testing.T) { assert.Equal(t, "GitHub", ProviderLabel("system.ai.github")) assert.Equal(t, "Genie One", ProviderLabel("SYSTEM.AI.genie_one_mcp")) diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index ead0a7b76c8..6c69b38f1ab 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -75,7 +75,8 @@ const ( // counted here: it ends the process before telemetry is uploaded. AitoolsErrorCategoryCommandInterrupted AitoolsErrorCategory = "COMMAND_INTERRUPTED" - // `aitools mcp`: profiles and sign-in. + // `aitools mcp`: profiles and sign-in. Bind failures send their category by value (mcp.BindCategory), so + // PROFILE_NOT_FOUND and PROFILE_HOST_CHANGED have no reference by name. AitoolsErrorCategoryNoProfile AitoolsErrorCategory = "NO_PROFILE" AitoolsErrorCategoryProfileRequired AitoolsErrorCategory = "PROFILE_REQUIRED" AitoolsErrorCategoryProfileNotFound AitoolsErrorCategory = "PROFILE_NOT_FOUND" @@ -84,7 +85,8 @@ const ( AitoolsErrorCategorySignInRequired AitoolsErrorCategory = "SIGN_IN_REQUIRED" AitoolsErrorCategorySignInTimeout AitoolsErrorCategory = "SIGN_IN_TIMEOUT" - // `aitools mcp`: the service list, agent configs and the proxy. + // `aitools mcp`: the service list, agent configs and the proxy. MCP_UPSTREAM_ERROR is in the schema but not sent yet: + // the proxy does not log upstream errors in v1. AitoolsErrorCategoryMCPServiceNotFound AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" AitoolsErrorCategoryMCPPermissionDenied AitoolsErrorCategory = "MCP_PERMISSION_DENIED" AitoolsErrorCategoryMCPLocationNotFound AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index 5546025d526..aae953d4ede 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -56,8 +56,9 @@ const ( type AitoolsTargetOutcome string const ( - AitoolsTargetOutcomeChanged AitoolsTargetOutcome = "CHANGED" - AitoolsTargetOutcomeUnchanged AitoolsTargetOutcome = "UNCHANGED" + AitoolsTargetOutcomeChanged AitoolsTargetOutcome = "CHANGED" + AitoolsTargetOutcomeUnchanged AitoolsTargetOutcome = "UNCHANGED" + // AitoolsTargetOutcomeUpdateAvailable mirrors the schema. No MCP command sends it yet. AitoolsTargetOutcomeUpdateAvailable AitoolsTargetOutcome = "UPDATE_AVAILABLE" AitoolsTargetOutcomeSkipped AitoolsTargetOutcome = "SKIPPED" AitoolsTargetOutcomeFailed AitoolsTargetOutcome = "FAILED" From 68debe476f495999552b13450a6376125eecd581 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:44:52 +0000 Subject: [PATCH 20/43] Refuse the MCP telemetry upload after the service rejects the credential A 401 that does not ask for a sign-in is answered with the auth failure, but the run's exit-time upload still authenticated with the credential the workspace had just rejected. The proxy now reports that answer to the command, which refuses the upload as a refresh failure does. Co-authored-by: Isaac --- cmd/aitools/mcp_telemetry.go | 5 +++++ cmd/aitools/mcp_telemetry_test.go | 10 +++++++++ libs/aitools/mcp/proxy/challenge.go | 6 +++++- libs/aitools/mcp/proxy/challenge_test.go | 27 +++++++++++++++++++++++- 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index ca94f89e33c..db8fc52b85b 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -248,6 +248,11 @@ func (t mcpProxyTelemetry) SignInPrompted(ctx context.Context, signedIn bool) { logMcpEvent(ctx, signInPromptedEvent(t.agent, t.service, signedIn)) } +// AuthRejected refuses the run's upload, since the upload would present the rejected credential again. +func (t mcpProxyTelemetry) AuthRejected(ctx context.Context) { + telemetry.RefuseUpload(ctx) +} + // signInPromptedEvent is the SIGN_IN_PROMPTED event of serve: the sign-in is required, and the // sign-in state is ACTIVE when the wait ended with the credential, PENDING otherwise. func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn bool) protos.AitoolsMcpEvent { diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index ccd28d3e870..4133aa14c64 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -218,6 +218,16 @@ func TestLogMcpEventDropsRejectedCredential(t *testing.T) { assert.False(t, uploads, "the add event is logged, and it is not uploaded with the rejected run") } +// TestMcpProxyAuthRejectedRefusesUpload checks that a 401 the proxy answers refuses the run's upload, even though the +// serve command sets its upload config at exit. +func TestMcpProxyAuthRejectedRefusesUpload(t *testing.T) { + ctx := telemetry.WithNewLogger(t.Context()) + telemetry.SetUploadConfig(ctx, &config.Config{Profile: "logfood", Host: "https://adb-1.test"}) + mcpProxyTelemetry{}.AuthRejected(ctx) + _, uploads := telemetry.UploadConfig(ctx) + assert.False(t, uploads) +} + func TestSignInPromptedEvent(t *testing.T) { for signedIn, state := range map[bool]protos.AitoolsMcpSignInState{ true: protos.AitoolsMcpSignInStateActive, diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index a5ff5a77dea..676b057c1ae 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -57,6 +57,9 @@ type Recorder interface { // earlier cancelled call opened, which the user may still have open. signedIn reports whether the credential became // ACTIVE. A retry after a cancelled call reports its own outcome. SignInPrompted(ctx context.Context, signedIn bool) + // AuthRejected runs when the service answers 401 without asking for a sign-in: the Databricks credential was + // rejected, and the answer is the auth failure. + AuthRejected(ctx context.Context) } // Challenge turns on sign-in handling for a service that needs a provider sign-in. @@ -159,7 +162,7 @@ func (h *signInHooks) Local(ctx context.Context, request []byte, notify func([]b // answer becomes a sign-in message, followed by tools/list_changed so an agent that cached its list re-lists. A 401 that // is not a challenge becomes the auth failure (see authFailure). Only the response to the request is read; any other // message passes through untouched. -func (h *signInHooks) Result(_ context.Context, request []byte, status int, message []byte) ([][]byte, error) { +func (h *signInHooks) Result(ctx context.Context, request []byte, status int, message []byte) ([][]byte, error) { req, err := parseMessage(request) if err != nil { return nil, err @@ -177,6 +180,7 @@ func (h *signInHooks) Result(_ context.Context, request []byte, status int, mess } challenged := h.challenged(&reply) if status == http.StatusUnauthorized && !challenged { + h.c.Recorder.AuthRejected(ctx) return [][]byte{errorResponse(req.ID, codeUpstream, h.authFailure())}, nil } if req.Method != methodToolsList && req.Method != methodToolsCall { diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index b7a46e0f550..7f4cbb90d66 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -68,10 +68,12 @@ func (s *signInStub) requests() []SignInRequest { // loginCommand is the hint that the command builds for a pinned config file and a profile name with a space. const loginCommand = `DATABRICKS_CONFIG_FILE='/work/cfg/my file.cfg' databricks aitools mcp login system.ai.github --profile 'my profile'` -// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. +// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. It counts the +// rejections of the Databricks credential too. type promptRecorder struct { mu sync.Mutex signedIn []bool + rejected int } func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { @@ -80,6 +82,12 @@ func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { r.signedIn = append(r.signedIn, signedIn) } +func (r *promptRecorder) AuthRejected(_ context.Context) { + r.mu.Lock() + defer r.mu.Unlock() + r.rejected++ +} + func (r *promptRecorder) prompts() []bool { r.mu.Lock() defer r.mu.Unlock() @@ -1022,6 +1030,23 @@ func TestUnauthorizedIsAuthFailureUnlessItAsksForSignIn(t *testing.T) { } } +// TestUnauthorizedRecordsRejectedCredential checks that a 401 without a sign-in challenge is recorded as a rejected +// credential, once per answer, so that the command refuses its telemetry upload. +func TestUnauthorizedRecordsRejectedCredential(t *testing.T) { + srv, _ := newFakeEndpoint(t, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + }) + signIn := &signInStub{reply: func(int) (bool, error) { return false, nil }} + recorder := &promptRecorder{} + challenge := testChallenge(signIn.SignIn) + challenge.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(challenge, io.Discard) + runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") + + assert.Equal(t, 1, recorder.rejected) +} + // TestRejectedCredentialDoesNotAskForLogout checks a rejected Databricks token after "Already signed in": the credential reads // as signed in, and the service rejects the token with a 401. The 401 is an auth failure, not a challenge, so the next // authenticate runs the sign-in as usual, and the logout answer is not given. From 380ca19a7e7f3b281952ae27d6f66cc5fad04c80 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:46:10 +0000 Subject: [PATCH 21/43] Keep the telemetry upload's host metadata failures out of the command output The SDK logs a failed host metadata lookup as a warning through its global logger, and the upload's resolution runs on a background context that reaches it. A command that succeeded then printed a warning that only its telemetry upload caused. The upload's resolver now logs the failure at debug and falls back to the user config, as the SDK does. Co-authored-by: Isaac --- cmd/aitools/mcp_telemetry_test.go | 26 ++++++++++++++++++++++++++ libs/aitools/mcp/binding.go | 23 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 4133aa14c64..e388ca81568 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -1,8 +1,11 @@ package aitools import ( + "bytes" + "context" "encoding/json" "errors" + "log/slog" "maps" "path/filepath" "slices" @@ -218,6 +221,29 @@ func TestLogMcpEventDropsRejectedCredential(t *testing.T) { assert.False(t, uploads, "the add event is logged, and it is not uploaded with the rejected run") } +// TestSetUploadConfigLogsNoWarningForFailedHostMetadata checks that an upload whose host metadata lookup fails warns +// nothing through the global logger, since the SDK would log that failure there. The command has succeeded, and the +// upload is best-effort. +func TestSetUploadConfigLogsNoWarningForFailedHostMetadata(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + factory := config.DefaultHostMetadataResolverFactory + config.DefaultHostMetadataResolverFactory = func(*config.Config) config.HostMetadataResolver { + return func(context.Context, string) (*config.HostMetadata, error) { return nil, errors.New("no such host") } + } + t.Cleanup(func() { config.DefaultHostMetadataResolverFactory = factory }) + var warnings bytes.Buffer + defaultLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&warnings, &slog.HandlerOptions{Level: slog.LevelWarn}))) + t.Cleanup(func() { slog.SetDefault(defaultLogger) }) + + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + setUploadConfig(ctx, "logfood", filepath.Join(home, ".databrickscfg")) + _, uploads := telemetry.UploadConfig(ctx) + assert.True(t, uploads) + assert.Empty(t, warnings.String()) +} + // TestMcpProxyAuthRejectedRefusesUpload checks that a 401 the proxy answers refuses the run's upload, even though the // serve command sets its upload config at exit. func TestMcpProxyAuthRejectedRefusesUpload(t *testing.T) { diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 6f2f83cc595..fc7cfad984d 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/databricks-sdk-go/config" ) @@ -120,9 +121,31 @@ func BoundConfigWithin(profileName, configFile string, budget time.Duration) (*c func BoundUploadConfig(profileName, configFile string, budget time.Duration) (*config.Config, error) { cfg := withinBudget(profileName, configFile, budget) cfg.Credentials = auth.CredentialsWithoutRefresh() + quietHostMetadata(cfg) return resolveBound(cfg) } +// quietHostMetadata makes a failed host metadata lookup of cfg a debug message. The SDK logs that failure as a warning +// through its global logger, since the resolution runs on a background context, so a command that succeeded would print a +// warning that only its telemetry upload caused. The SDK falls back to the user config when the lookup fails, and so does +// this one, with a nil result. +func quietHostMetadata(cfg *config.Config) { + resolve := cfg.DefaultHostMetadataResolver() + if factory := config.DefaultHostMetadataResolverFactory; factory != nil { + if r := factory(cfg); r != nil { + resolve = r + } + } + cfg.HostMetadataResolver = func(ctx context.Context, host string) (*config.HostMetadata, error) { + meta, err := resolve(ctx, host) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve host metadata for %s: %s", host, err) + return nil, nil + } + return meta, nil + } +} + // withinBudget is the config of profileName in configFile, with the SDK's request and retry budgets set to budget. func withinBudget(profileName, configFile string, budget time.Duration) *config.Config { seconds := int((budget + time.Second - 1) / time.Second) From f016d6e62c4c25f835207d976413b6d12d491030 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:49:25 +0000 Subject: [PATCH 22/43] Group the MCP telemetry profile count as mcp list does, and tidy the MCP helpers boundProfileCount now groups servers through the same helper mcp list uses, so the count matches the listing in every case, including a config file that does not resolve. KnownLeaves is marked as kept for its tests, the sign-in constructors return their command directly, the pruned-install test setup is a named helper, and the enum comments say that the schema mirrors are not sent by the CLI. Co-authored-by: Isaac --- cmd/aitools/mcp_list.go | 18 ++++++++++++------ cmd/aitools/mcp_login.go | 6 ++---- cmd/aitools/mcp_remove_test.go | 17 ++++++++++------- cmd/aitools/mcp_telemetry.go | 13 +++++-------- libs/aitools/mcp/known.go | 2 ++ libs/aitools/mcp/proxy/challenge_test.go | 4 ++-- libs/aitools/mcp/proxy/run_test.go | 2 +- libs/telemetry/protos/aitools_install.go | 8 ++++---- 8 files changed, 38 insertions(+), 32 deletions(-) diff --git a/cmd/aitools/mcp_list.go b/cmd/aitools/mcp_list.go index 8d1b6ce2e67..b5015bfac74 100644 --- a/cmd/aitools/mcp_list.go +++ b/cmd/aitools/mcp_list.go @@ -134,6 +134,16 @@ type profileKey struct { path string } +// profileGroup returns the group of a server entry in list: its profile in the config file that check resolved, or in the +// file as written when that file does not resolve. +func profileGroup(e mcp.Entry, check mcp.EntryCheck) profileKey { + path := check.Path + if path == "" { + path = e.ConfigFile + } + return profileKey{profile: e.Profile, path: path} +} + // NewMcpListCmd returns `aitools mcp list`. Without --check it reads the state file and the agent configs only, and // makes no network calls. Drift and unreadable entries are reported, not failed: the command exits 0. func NewMcpListCmd() *cobra.Command { @@ -247,12 +257,8 @@ func buildMcpList(ctx context.Context, profileName string) (mcpListOutput, error warnedFiles := map[string]bool{} for i, e := range entries { check := checks[i] - path := check.Path - if path == "" { - // The file cannot be resolved, so the entry is grouped under its file as written. - path = e.ConfigFile - } - key := profileKey{profile: e.Profile, path: path} + key := profileGroup(e, check) + path := key.path p, ok := groups[key] if !ok { p = &mcpListProfile{Profile: e.Profile, Host: e.Host, Status: statusOK, Servers: []mcpListServer{}, configPath: path} diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index 0423e9db8fd..99b38f4fbc6 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -56,7 +56,7 @@ type mcpSignInAction func(ctx context.Context, errOut io.Writer, w *databricks.W // NewMcpLoginCmd returns `aitools mcp login`. func NewMcpLoginCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "login ", Short: "Sign in to a Databricks managed MCP service with your account", Long: `Sign in to a Databricks managed MCP service that uses your account with an external provider, such @@ -71,12 +71,11 @@ sign-in to complete. Without a browser, open the printed URL yourself. return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } - return cmd } // NewMcpLogoutCmd returns `aitools mcp logout`. func NewMcpLogoutCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "logout ", Short: "Remove your sign-in to a Databricks managed MCP service", Long: `Remove your sign-in to a Databricks managed MCP service on a profile. The install of the server @@ -90,7 +89,6 @@ stays in place. Running it when you are not signed in is not an error. return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } - return cmd } // runMcpSignInCommand resolves the profile for service and runs action. In JSON mode the document is diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index f2f82b84ed9..9267a9ae0ec 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -1116,18 +1116,21 @@ func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { assert.Contains(t, stdout.String(), `"removed"`) } +// seedPrunedInstall records an install of genie_one_mcp for logfood in claude-code whose entry the agent config no longer holds. +func seedPrunedInstall(t *testing.T, claudePath string) { + t.Helper() + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + require.NoError(t, os.WriteFile(claudePath, []byte(`{"mcpServers":{}}`), 0o600)) +} + // TestMcpRemovePrunedEntryReportsChanged checks that an entry the agent config no longer holds is reported CHANGED, since // its record is dropped, and FAILED when the state cannot be saved, as a record-only removal is. func TestMcpRemovePrunedEntryReportsChanged(t *testing.T) { home := withHome(t) - claude := agents.ByName("claude-code") claudePath := filepath.Join(home, ".claude.json") - seedPruned := func() { - saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) - require.NoError(t, os.WriteFile(claudePath, []byte(`{"mcpServers":{}}`), 0o600)) - } - seedPruned() + seedPrunedInstall(t, claudePath) var stdout, logs bytes.Buffer remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") require.NoError(t, err) @@ -1137,7 +1140,7 @@ func TestMcpRemovePrunedEntryReportsChanged(t *testing.T) { require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) assert.Equal(t, protos.AitoolsTargetOutcomeChanged, events[0].AitoolsMcpEvent.Servers[0].Outcome) - seedPruned() + seedPrunedInstall(t, claudePath) saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } t.Cleanup(func() { saveMCPState = mcp.SaveState }) stdout.Reset() diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index db8fc52b85b..35f88327a0d 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -100,16 +100,13 @@ func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) bool { return true } -// boundProfileCount returns the number of distinct profiles that have servers installed. A profile is its name in its -// config file, as list groups them, so DEFAULT in two config files counts twice. +// boundProfileCount returns the number of distinct profiles that have servers installed, grouped as list groups them +// (profileGroup), so DEFAULT in two config files counts twice. func boundProfileCount(ctx context.Context, state *mcp.State) int64 { profiles := map[profileKey]bool{} - for _, e := range state.Servers { - path, err := mcp.ResolveConfigFile(ctx, e.ConfigFile) - if err != nil { - path = e.ConfigFile - } - profiles[profileKey{profile: e.Profile, path: path}] = true + checks := mcp.CheckEntries(ctx, state) + for i, e := range state.Servers { + profiles[profileGroup(e, checks[i])] = true } return int64(len(profiles)) } diff --git a/libs/aitools/mcp/known.go b/libs/aitools/mcp/known.go index 10bf6294947..2ec0f01b62f 100644 --- a/libs/aitools/mcp/known.go +++ b/libs/aitools/mcp/known.go @@ -66,6 +66,8 @@ var knownSystemAI = map[string]KnownService{ } // KnownLeaves returns the leaf names of the system.ai services in the CLI's table, sorted. +// +//deadcode:allow used by the service-mapping tests func KnownLeaves() []string { return slices.Sorted(maps.Keys(knownSystemAI)) } diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 7f4cbb90d66..48f3ab8405c 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -616,7 +616,7 @@ func TestChallengeMatchesServiceNameWithoutCase(t *testing.T) { `{"mode":"url","url":"` + testURL + `"}]}}}` srv, _ := newFakeEndpoint(t, answerWith(http.StatusOK, body)) b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: "SYSTEM.AI.GITHUB", Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: "SYSTEM.AI.GITHUB", Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, Recorder: &promptRecorder{}}, io.Discard) out, _ := runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") require.Len(t, out, 1) @@ -1087,7 +1087,7 @@ func TestAuthFailureNamesPinnedConfigFile(t *testing.T) { http.Error(w, "unauthorized", http.StatusUnauthorized) }) b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, PinnedFile: "/work/cfg/my file.cfg"}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, PinnedFile: "/work/cfg/my file.cfg", Recorder: &promptRecorder{}}, io.Discard) out, _ := runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") require.Len(t, out, 1) diff --git a/libs/aitools/mcp/proxy/run_test.go b/libs/aitools/mcp/proxy/run_test.go index 8a8f195027d..f4020a9d3a1 100644 --- a/libs/aitools/mcp/proxy/run_test.go +++ b/libs/aitools/mcp/proxy/run_test.go @@ -100,7 +100,7 @@ func TestRunAdvertisesListChangedOnlyForSignInServices(t *testing.T) { assert.Contains(t, out.String(), `"listChanged":false`) opts, out, _ = runOptions(t, srv.URL, cfg, initialize) - opts.Challenge = &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub"} + opts.Challenge = &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", Recorder: &promptRecorder{}} require.NoError(t, Run(t.Context(), opts)) assert.Contains(t, out.String(), `"listChanged":true`) } diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index 6c69b38f1ab..13c700bf24d 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -75,8 +75,8 @@ const ( // counted here: it ends the process before telemetry is uploaded. AitoolsErrorCategoryCommandInterrupted AitoolsErrorCategory = "COMMAND_INTERRUPTED" - // `aitools mcp`: profiles and sign-in. Bind failures send their category by value (mcp.BindCategory), so - // PROFILE_NOT_FOUND and PROFILE_HOST_CHANGED have no reference by name. + // `aitools mcp`: profiles and sign-in. Bind failures send their category by value (mcp.BindCategory). PROFILE_NOT_FOUND + // and PROFILE_HOST_CHANGED mirror the schema, and no CLI command names them. AitoolsErrorCategoryNoProfile AitoolsErrorCategory = "NO_PROFILE" AitoolsErrorCategoryProfileRequired AitoolsErrorCategory = "PROFILE_REQUIRED" AitoolsErrorCategoryProfileNotFound AitoolsErrorCategory = "PROFILE_NOT_FOUND" @@ -85,8 +85,8 @@ const ( AitoolsErrorCategorySignInRequired AitoolsErrorCategory = "SIGN_IN_REQUIRED" AitoolsErrorCategorySignInTimeout AitoolsErrorCategory = "SIGN_IN_TIMEOUT" - // `aitools mcp`: the service list, agent configs and the proxy. MCP_UPSTREAM_ERROR is in the schema but not sent yet: - // the proxy does not log upstream errors in v1. + // `aitools mcp`: the service list, agent configs and the proxy. MCP_UPSTREAM_ERROR mirrors the schema, and no CLI + // command sends it yet: the proxy does not log upstream errors in v1. AitoolsErrorCategoryMCPServiceNotFound AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" AitoolsErrorCategoryMCPPermissionDenied AitoolsErrorCategory = "MCP_PERMISSION_DENIED" AitoolsErrorCategoryMCPLocationNotFound AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" From 4016b82ebfa33a219132624db3a82a2b25764e24 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Mon, 16 Nov 2026 13:15:39 +0000 Subject: [PATCH 23/43] Read the removed servers through jsonServers in the telemetry tests Card 01 drops a container that a removal leaves empty, so .mcpServers is gone after the last server is removed. Two remove tests that this card adds read it through jsonServers, as card 05's remove tests do. Co-authored-by: Isaac --- cmd/aitools/mcp_remove_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 9267a9ae0ec..784496dc32b 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -202,7 +202,7 @@ func TestMcpRemoveMakesNoRequestWhenTelemetryIsOff(t *testing.T) { _, _, err := runMcpCmd(t, "remove", "--names", "genie_one_mcp-logfood", "--output", "json") require.NoError(t, err) - assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood") + assert.NotContains(t, jsonServers(t, claudePath), "genie_one_mcp-logfood") } func TestMcpRemoveSkipsEntryChangedOutsideCLI(t *testing.T) { @@ -1058,7 +1058,7 @@ func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { require.Len(t, telemetry.BufferedEvents(remove.Context()), 1, "the removal is logged") _, uploads := telemetry.UploadConfig(remove.Context()) assert.False(t, uploads, "a drifted profile is not used to upload") - assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood", "the removal is done") + assert.NotContains(t, jsonServers(t, claudePath), "genie_one_mcp-logfood", "the removal is done") } // TestMcpRemoveUploadUsesMatchingProfile checks that a profile whose recorded servers still match is used for the upload, From 8548dbf10267210c4904935d60c562c3b192f7c1 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Mon, 16 Nov 2026 14:44:32 +0000 Subject: [PATCH 24/43] Regenerate the serve telemetry golden for the Error prefix Card 06b prints serve's binding failures as `Error: `, like every other failure, so the failed start in the mcp-serve telemetry golden gets the prefix. Co-authored-by: Isaac --- acceptance/cmd/aitools/telemetry/mcp-serve/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt index d8fec5e3dcf..83b979b1ad4 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -17,7 +17,7 @@ No install record for system.ai.genie_one_mcp on profile "logfood"; using its cu === A binding that fails logs no PROXY_STARTED, so no upload follows >>> musterr [CLI] aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex -MCP server system.ai.genie_one_mcp needs Databricks profile "gone", which does not exist in [TEST_TMP_DIR]/.databrickscfg. Set it up with: databricks auth login --profile gone +Error: MCP server system.ai.genie_one_mcp needs Databricks profile "gone", which does not exist in [TEST_TMP_DIR]/.databrickscfg. Set it up with: databricks auth login --profile gone === authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait { From f2b76175e7fa7066aa71f1f43d05d1631dc87660 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 10:45:10 +0000 Subject: [PATCH 25/43] Adapt telemetry to the failure helper from card 04 Card 04 dropped the cause argument of failure. The telemetry tests and the duplicate-name refusal that card 04 added now use the protos category names that this card introduces. Co-authored-by: Isaac --- cmd/aitools/mcp_add.go | 2 +- cmd/aitools/mcp_telemetry_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index 4ae3d902930..8f8b2327d4d 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -903,7 +903,7 @@ func planAdds(ctx context.Context, in planInput, state *mcp.State, fullNames []s } // A name recovered from an agent entry can repeat across services only through a hand edit; two plans must not share it. if chosen[name] { - return nil, failure(categoryNameConflict, "%s and another requested server both use %s; remove one of the entries first", fullName, name) + return nil, failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s and another requested server both use %s; remove one of the entries first", fullName, name) } chosen[name] = true taken[name] = true diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index e388ca81568..d6e4e68b106 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -150,7 +150,7 @@ func TestSignInServerResult(t *testing.T) { { // A run that never learned the state, such as a read that failed, reports none. name: "state not learned", - err: failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "not found"), + err: failure(protos.AitoolsErrorCategoryMCPServiceNotFound, "not found"), want: protos.AitoolsMcpServerResult{ Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, ErrorCategory: protos.AitoolsErrorCategoryMCPServiceNotFound, @@ -159,7 +159,7 @@ func TestSignInServerResult(t *testing.T) { { name: "timed out", state: signInStateTimeout, - err: failure(protos.AitoolsErrorCategorySignInTimeout, nil, "timed out"), + err: failure(protos.AitoolsErrorCategorySignInTimeout, "timed out"), want: protos.AitoolsMcpServerResult{ Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, ErrorCategory: protos.AitoolsErrorCategorySignInTimeout, SignInState: protos.AitoolsMcpSignInStatePending, From d32c9a2cc905b0b336da0dde7021d54a51b97a07 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 14:48:02 +0000 Subject: [PATCH 26/43] Adapt the name-conflict check to the telemetry error enum The card 04 review's name check in chooseName uses the category that telemetry now names in the protos enum, since this card removed the local error categories. The test that checks an unreadable config uses the same enum value. Co-authored-by: Isaac --- cmd/aitools/mcp_add.go | 2 +- cmd/aitools/mcp_add_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index 8f8b2327d4d..563ea34439d 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -961,7 +961,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st } // The agents must hold the state's name, or the run would add a second name for the same server beside theirs. if slices.ContainsFunc(owned, func(name string) bool { return name != stored }) { - return "", nil, "", failure(categoryNameConflict, "%s is installed on profile %s as %s, but agents hold it as %s; remove the extra entries", fullName, b.profile, stored, strings.Join(owned, ", ")) + return "", nil, "", failure(protos.AitoolsErrorCategoryMCPNameConflict, "%s is installed on profile %s as %s, but agents hold it as %s; remove the extra entries", fullName, b.profile, stored, strings.Join(owned, ", ")) } } else { switch len(owned) { diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index 210b5eec64d..735c27b0eef 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -963,7 +963,7 @@ func TestMcpAddRefusesUnreadableDetectedAgentBeforeAnyRequest(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github") require.Error(t, err) - assert.Equal(t, categoryAgentConfigInvalid, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryAgentConfigInvalid, errorCategory(err)) } // TestValidateMCPAddFlagsRefusesAgentsWithNoName checks that --agents holding no agent name is refused, rather than read as From 26ab9b25f49b1f270ed4b389e4b5677ff83ba0f4 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 15:04:32 +0000 Subject: [PATCH 27/43] Regenerate goldens for the GovCloud GitHub app in the service note Card 02 names the AWS GovCloud app in the GitHub note, which add prints. Co-authored-by: Isaac --- acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt index 46253de58e8..3137ec011a5 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt @@ -34,7 +34,7 @@ system.ai.genie_one_mcp does not need sign-in. >>> [CLI] aitools mcp login system.ai.github --profile logfood Already signed in to system.ai.github on profile logfood. -Public repositories only until a repository owner installs https://github.com/apps/databricks-github-connector. +Public repositories only until a repository owner installs https://github.com/apps/databricks-github-connector (on AWS GovCloud, https://github.com/apps/databricks-gh-connector-gov-cloud). { "calling_agent": "TYPE_UNSPECIFIED", "operation": "SIGN_IN", From 20eb03a5b2ecf8903a34273aaf7fe5fe43508ff0 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 18:21:59 +0000 Subject: [PATCH 28/43] Move the debug-only log handler to libs/log/handler The telemetry hooks need the handler that logs a Warn at Debug, and libs/aitools/mcp cannot reach the unexported copy in cmd/aitools. Move it unchanged so both can use it. Co-authored-by: Isaac --- cmd/aitools/telemetry.go | 28 ++-------------------------- libs/log/handler/debug_only.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 libs/log/handler/debug_only.go diff --git a/cmd/aitools/telemetry.go b/cmd/aitools/telemetry.go index 4e451feb6eb..4a771c1022f 100644 --- a/cmd/aitools/telemetry.go +++ b/cmd/aitools/telemetry.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/log/handler" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go/config" @@ -83,7 +84,7 @@ func startTelemetryAuth(cmd *cobra.Command, args []string) *pendingAuth { if telemetry.Disabled(ctx) { return nil } - sctx := log.NewContext(ctx, slog.New(debugOnlyHandler{log.GetLogger(ctx).Handler()})) + sctx := log.NewContext(ctx, slog.New(handler.DebugOnlyHandler{Handler: log.GetLogger(ctx).Handler()})) shadow := &cobra.Command{Use: cmd.Use} shadow.Flags().AddFlagSet(cmd.Flags()) shadow.SetContext(root.SkipTokenRefresh(root.SkipLoadBundle(root.SkipPrompt(sctx)))) @@ -160,31 +161,6 @@ func (p *pendingAuth) finished() bool { } } -// debugOnlyHandler logs records above Debug (e.g. hostmetadata's "failed to -// fetch host metadata" Warn) at Debug, so they show only with --debug. -type debugOnlyHandler struct{ slog.Handler } - -func debugOnly(level slog.Level) slog.Level { - return min(level, log.LevelDebug) -} - -func (h debugOnlyHandler) Enabled(ctx context.Context, level slog.Level) bool { - return h.Handler.Enabled(ctx, debugOnly(level)) -} - -func (h debugOnlyHandler) Handle(ctx context.Context, r slog.Record) error { - r.Level = debugOnly(r.Level) - return h.Handler.Handle(ctx, r) -} - -func (h debugOnlyHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - return debugOnlyHandler{h.Handler.WithAttrs(attrs)} -} - -func (h debugOnlyHandler) WithGroup(name string) slog.Handler { - return debugOnlyHandler{h.Handler.WithGroup(name)} -} - // installOpts is the subset of install options telemetry records, kept narrow // so every field that leaves the machine is visible here. type installOpts struct { diff --git a/libs/log/handler/debug_only.go b/libs/log/handler/debug_only.go new file mode 100644 index 00000000000..364ad5a34ec --- /dev/null +++ b/libs/log/handler/debug_only.go @@ -0,0 +1,33 @@ +package handler + +import ( + "context" + "log/slog" + + "github.com/databricks/cli/libs/log" +) + +// DebugOnlyHandler logs records above Debug (e.g. hostmetadata's "failed to +// fetch host metadata" Warn) at Debug, so they show only with --debug. +type DebugOnlyHandler struct{ slog.Handler } + +func debugOnly(level slog.Level) slog.Level { + return min(level, log.LevelDebug) +} + +func (h DebugOnlyHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.Handler.Enabled(ctx, debugOnly(level)) +} + +func (h DebugOnlyHandler) Handle(ctx context.Context, r slog.Record) error { + r.Level = debugOnly(r.Level) + return h.Handler.Handle(ctx, r) +} + +func (h DebugOnlyHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return DebugOnlyHandler{h.Handler.WithAttrs(attrs)} +} + +func (h DebugOnlyHandler) WithGroup(name string) slog.Handler { + return DebugOnlyHandler{h.Handler.WithGroup(name)} +} From 17fc9724435ecc43a3ee578af3b8ae73a3783795 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 18:22:00 +0000 Subject: [PATCH 29/43] Demote the registered host metadata warning of the telemetry upload The registered factory logs a failed host metadata fetch as a Warn and returns no error, so the error branch that demoted it never ran in the shipped binary, and a successful remove still printed the warning. The lookup now runs with a debug-only logger of the command. The warning test stubs the factory the way the registered one behaves. Co-authored-by: Isaac --- cmd/aitools/mcp_telemetry.go | 2 +- cmd/aitools/mcp_telemetry_test.go | 7 ++++++- libs/aitools/mcp/binding.go | 19 ++++++++++++------- libs/aitools/mcp/binding_test.go | 4 ++-- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 35f88327a0d..f8fb458facb 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -20,7 +20,7 @@ func setUploadConfig(ctx context.Context, profileName, configFile string) { if telemetry.Disabled(ctx) { return } - cfg, err := mcp.BoundUploadConfig(profileName, configFile, telemetryAuthBudget) + cfg, err := mcp.BoundUploadConfig(ctx, profileName, configFile, telemetryAuthBudget) if err != nil { log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) return diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index d6e4e68b106..037c7b58c3d 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go/config" @@ -228,8 +229,12 @@ func TestSetUploadConfigLogsNoWarningForFailedHostMetadata(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") factory := config.DefaultHostMetadataResolverFactory + // Like the registered factory (libs/hostmetadata), the resolver logs its failure as a warning and returns no error. config.DefaultHostMetadataResolverFactory = func(*config.Config) config.HostMetadataResolver { - return func(context.Context, string) (*config.HostMetadata, error) { return nil, errors.New("no such host") } + return func(ctx context.Context, host string) (*config.HostMetadata, error) { + log.Warnf(ctx, "[hostmetadata] failed to fetch host metadata for %s, will skip for 1m0s", host) + return nil, nil + } } t.Cleanup(func() { config.DefaultHostMetadataResolverFactory = factory }) var warnings bytes.Buffer diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index fc7cfad984d..5337a1585dd 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log/slog" "net/http" "path/filepath" "strings" @@ -16,6 +17,7 @@ import ( "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/log/handler" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/databricks-sdk-go/config" ) @@ -118,28 +120,31 @@ func BoundConfigWithin(profileName, configFile string, budget time.Duration) (*c // BoundUploadConfig is BoundConfigWithin for a config that only a telemetry upload uses. Its credentials never refresh a // token, since the command may exit before a refresh finishes, and a rotated single-use refresh token would be lost (see // auth.CredentialsWithoutRefresh). It does not authenticate. -func BoundUploadConfig(profileName, configFile string, budget time.Duration) (*config.Config, error) { +func BoundUploadConfig(ctx context.Context, profileName, configFile string, budget time.Duration) (*config.Config, error) { cfg := withinBudget(profileName, configFile, budget) cfg.Credentials = auth.CredentialsWithoutRefresh() - quietHostMetadata(cfg) + quietHostMetadata(ctx, cfg) return resolveBound(cfg) } // quietHostMetadata makes a failed host metadata lookup of cfg a debug message. The SDK logs that failure as a warning // through its global logger, since the resolution runs on a background context, so a command that succeeded would print a -// warning that only its telemetry upload caused. The SDK falls back to the user config when the lookup fails, and so does +// warning that only its telemetry upload caused. The registered factory (libs/hostmetadata) logs its own failure as a +// warning and returns no error, so the lookup runs with a debug-only logger of ctx's command, and the error case below is +// the fallback for a resolver that returns one. The SDK falls back to the user config when the lookup fails, and so does // this one, with a nil result. -func quietHostMetadata(cfg *config.Config) { +func quietHostMetadata(ctx context.Context, cfg *config.Config) { resolve := cfg.DefaultHostMetadataResolver() if factory := config.DefaultHostMetadataResolverFactory; factory != nil { if r := factory(cfg); r != nil { resolve = r } } - cfg.HostMetadataResolver = func(ctx context.Context, host string) (*config.HostMetadata, error) { - meta, err := resolve(ctx, host) + logger := slog.New(handler.DebugOnlyHandler{Handler: log.GetLogger(ctx).Handler()}) + cfg.HostMetadataResolver = func(sdkCtx context.Context, host string) (*config.HostMetadata, error) { + meta, err := resolve(log.NewContext(sdkCtx, logger), host) if err != nil { - log.Debugf(ctx, "telemetry: cannot resolve host metadata for %s: %s", host, err) + log.Debugf(sdkCtx, "telemetry: cannot resolve host metadata for %s: %s", host, err) return nil, nil } return meta, nil diff --git a/libs/aitools/mcp/binding_test.go b/libs/aitools/mcp/binding_test.go index 5b5f0f48af1..785cbc270c2 100644 --- a/libs/aitools/mcp/binding_test.go +++ b/libs/aitools/mcp/binding_test.go @@ -952,11 +952,11 @@ func TestConfigFileCommandUsesForwardSlashes(t *testing.T) { func TestBoundUploadConfigNeverRefreshes(t *testing.T) { cfgPath := bindingTestConfig(t, bindingTestProfileContents(bindingTestToken)) - upload, err := mcp.BoundUploadConfig(bindingTestProfile, cfgPath, time.Second) + upload, err := mcp.BoundUploadConfig(t.Context(), bindingTestProfile, cfgPath, time.Second) require.NoError(t, err) assert.Equal(t, auth.CredentialsWithoutRefresh(), upload.Credentials) session, err := mcp.BoundConfigWithin(bindingTestProfile, cfgPath, time.Second) require.NoError(t, err) - assert.NotEqual(t, auth.CredentialsWithoutRefresh(), session.Credentials) + assert.Nil(t, session.Credentials, "the session leaves Credentials unset, so the SDK keeps the default chain") } From 8d4081af9302fcbd56a4ec4f62516da4a9c8eabb Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 18:22:02 +0000 Subject: [PATCH 30/43] Guard the telemetry logger's refusal flag and events with a mutex The MCP proxy runs its hooks on one goroutine per request, so two rejected requests wrote the refusal flag, and events appended, concurrently. The mutex covers both, and Upload reads a snapshot of the events. Co-authored-by: Isaac --- libs/telemetry/logger.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index 97ffad8cd30..b534e81a9b9 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "strconv" + "sync" "sync/atomic" "time" "uuid" @@ -50,6 +51,8 @@ func SetUploadConfig(ctx context.Context, cfg *config.Config) { // rejected, since the upload would present that credential again. Like Log, it is a no-op without a logger. func RefuseUpload(ctx context.Context) { if l, ok := loggerFromContext(ctx); ok { + l.mu.Lock() + defer l.mu.Unlock() l.refused = true } } @@ -58,7 +61,13 @@ func RefuseUpload(ctx context.Context) { // cmdctx.ConfigUsed holds. It returns false when the run refused its upload or has no config. func UploadConfig(ctx context.Context) (*config.Config, bool) { l, ok := loggerFromContext(ctx) - if !ok || l.refused { + if !ok { + return nil, false + } + l.mu.Lock() + refused := l.refused + l.mu.Unlock() + if refused { return nil, false } if l.upload != nil { @@ -84,6 +93,8 @@ func Log(ctx context.Context, event protos.DatabricksCliLog) { } type logger struct { + // mu guards logs and refused, which the goroutines of a command's MCP proxy write concurrently. + mu sync.Mutex logs []protos.FrontendLog stateLineageHash string @@ -114,6 +125,8 @@ func StateLineageHash(ctx context.Context) string { } func (l *logger) log(event protos.DatabricksCliLog) { + l.mu.Lock() + defer l.mu.Unlock() l.logs = append(l.logs, protos.FrontendLog{ FrontendLogEventID: uuid.New().String(), Entry: protos.FrontendLogEntry{ @@ -136,7 +149,10 @@ const ( func Upload(ctx context.Context, ec protos.ExecutionContext) error { l := fromContext(ctx) - if len(l.logs) == 0 { + l.mu.Lock() + logs := l.logs + l.mu.Unlock() + if len(logs) == 0 { log.Debugf(ctx, "no telemetry logs to upload") return nil } @@ -148,12 +164,12 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { } // Set the execution context for all logs. - for i := range l.logs { - l.logs[i].Entry.DatabricksCliLog.ExecutionContext = &ec + for i := range logs { + logs[i].Entry.DatabricksCliLog.ExecutionContext = &ec } - protoLogs := make([]string, len(l.logs)) - for i, log := range l.logs { + protoLogs := make([]string, len(logs)) + for i, log := range logs { b, err := json.Marshal(log) if err != nil { return fmt.Errorf("failed to marshal log: %w", err) @@ -174,7 +190,7 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { return errors.New("a previous telemetry upload is still running") } - timeout, err := uploadTimeout(ctx, l.logs) + timeout, err := uploadTimeout(ctx, logs) if err != nil { return err } From 0998aa5618a67ee76cf04ae3e5aca39e90ca721e Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 18:22:03 +0000 Subject: [PATCH 31/43] Correct the serve and add test comments on the upload config The serve comment said a refused upload sets no config, but the config is still resolved and ignored. The add test helper comment said resolving the upload config makes no request, which the host metadata lookup does. Co-authored-by: Isaac --- cmd/aitools/mcp_add_test.go | 2 +- cmd/aitools/mcp_serve.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index 735c27b0eef..3f33a457792 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -314,7 +314,7 @@ func withArg0(t *testing.T, arg0 string) { // noRequestServer returns a server that fails the test on any request. A command that refuses before // it contacts the profile's host never reaches it, with telemetry on or off: the upload's config is resolved only after -// the binding checks have passed, and that resolution makes no request. +// the binding checks have passed, and resolving it is what makes the host metadata request. func noRequestServer(t *testing.T) *httptest.Server { t.Helper() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index d887b2af71c..dc7368c7be0 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -284,7 +284,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { Challenge: challenge, }) // The upload authenticates with the profile at exit, with a config of its own that never refreshes a token, since the - // session's config refreshes its token. A refresh failure has refused the upload (proxyResult), so nothing is set then. + // session's config refreshes its token. A refused upload (proxyResult) still resolves this config, and Upload ignores it. setUploadConfig(cmd.Context(), profileName, cfg.ConfigFile) return err } From 628dabce382b1a02d1b984a0eb9976a286db9f0c Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 18:38:28 +0000 Subject: [PATCH 32/43] Lock the telemetry events read by BufferedEvents and Upload BufferedEvents read the events without the mutex that log appends under, and Upload set the execution context on the shared events. Both now read a locked copy. Co-authored-by: Isaac --- libs/telemetry/logger.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index b534e81a9b9..32a228cb404 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "slices" "strconv" "sync" "sync/atomic" @@ -149,8 +150,9 @@ const ( func Upload(ctx context.Context, ec protos.ExecutionContext) error { l := fromContext(ctx) + // Upload works on a copy, so that setting the execution context does not write the events a proxy goroutine may append to. l.mu.Lock() - logs := l.logs + logs := slices.Clone(l.logs) l.mu.Unlock() if len(logs) == 0 { log.Debugf(ctx, "no telemetry logs to upload") @@ -402,6 +404,8 @@ func BufferedEvents(ctx context.Context) []protos.DatabricksCliLog { if !ok { return nil } + l.mu.Lock() + defer l.mu.Unlock() out := make([]protos.DatabricksCliLog, len(l.logs)) for i, fl := range l.logs { out[i] = fl.Entry.DatabricksCliLog From 4455ed3f2484da102c59c785aa9b8af18988b2cd Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 18:38:29 +0000 Subject: [PATCH 33/43] Test the debug-only log handler The handler is shared by the telemetry auth and upload paths, so its level clamp and its wrappers get their own tests. Co-authored-by: Isaac --- libs/log/handler/debug_only_test.go | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 libs/log/handler/debug_only_test.go diff --git a/libs/log/handler/debug_only_test.go b/libs/log/handler/debug_only_test.go new file mode 100644 index 00000000000..527cbe71a63 --- /dev/null +++ b/libs/log/handler/debug_only_test.go @@ -0,0 +1,30 @@ +package handler + +import ( + "bytes" + "context" + "log/slog" + "testing" + + "github.com/databricks/cli/libs/log" + "github.com/stretchr/testify/assert" +) + +func TestDebugOnlyHandlerLogsWarningsAtDebug(t *testing.T) { + var buf bytes.Buffer + base := slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: log.LevelDebug}) + logger := slog.New(DebugOnlyHandler{Handler: base}.WithAttrs([]slog.Attr{slog.String("k", "v")}).(DebugOnlyHandler).WithGroup("g")) + + logger.Warn("failed to fetch") + assert.Contains(t, buf.String(), "level=DEBUG") + assert.NotContains(t, buf.String(), "level=WARN") +} + +func TestDebugOnlyHandlerEnabledClampsLevel(t *testing.T) { + debugBase := slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: log.LevelDebug}) + assert.True(t, DebugOnlyHandler{Handler: debugBase}.Enabled(context.Background(), slog.LevelWarn)) + + // A warning is still filtered when the base handler's level is above Debug. + errorBase := slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelError}) + assert.False(t, DebugOnlyHandler{Handler: errorBase}.Enabled(context.Background(), slog.LevelWarn)) +} From f9a7f6b5eaf66ed72e56b55ad33550231ff76ca6 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 18:38:30 +0000 Subject: [PATCH 34/43] Shorten the host metadata comment and fix the offline remove test comment The comment on quietHostMetadata repeated the reasons across seven lines. The remove test comment said the upload config makes no request, which the host metadata lookup does. Co-authored-by: Isaac --- cmd/aitools/mcp_remove_test.go | 2 +- libs/aitools/mcp/binding.go | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 784496dc32b..2238e30edf2 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -1100,7 +1100,7 @@ func TestMcpRemoveUploadIgnoresEnvironmentToken(t *testing.T) { } // TestMcpRemoveOfflineSetsUploadWithoutWarning checks that a remove that cannot reach the workspace warns nothing: the -// upload's config is resolved without a request, so there is no failure to report. +// upload's config is resolved, and its host metadata lookup fails quietly, so there is no warning to report. func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 5337a1585dd..8c9f62c42d4 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -127,12 +127,10 @@ func BoundUploadConfig(ctx context.Context, profileName, configFile string, budg return resolveBound(cfg) } -// quietHostMetadata makes a failed host metadata lookup of cfg a debug message. The SDK logs that failure as a warning -// through its global logger, since the resolution runs on a background context, so a command that succeeded would print a -// warning that only its telemetry upload caused. The registered factory (libs/hostmetadata) logs its own failure as a -// warning and returns no error, so the lookup runs with a debug-only logger of ctx's command, and the error case below is -// the fallback for a resolver that returns one. The SDK falls back to the user config when the lookup fails, and so does -// this one, with a nil result. +// quietHostMetadata makes a failed host metadata lookup of cfg a debug message, so that a command which succeeded does not +// warn about its telemetry upload. The lookup runs with a debug-only logger of ctx, since the SDK's background context +// would send the registered resolver's warning to the global logger. A failed lookup falls back to the user config, as +// the SDK does. func quietHostMetadata(ctx context.Context, cfg *config.Config) { resolve := cfg.DefaultHostMetadataResolver() if factory := config.DefaultHostMetadataResolverFactory; factory != nil { From 2afef51c2dae935f876a1b55a24b3769104ece75 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 18:39:06 +0000 Subject: [PATCH 35/43] Use the test context in the debug-only handler test Co-authored-by: Isaac --- libs/log/handler/debug_only_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libs/log/handler/debug_only_test.go b/libs/log/handler/debug_only_test.go index 527cbe71a63..3a313997d82 100644 --- a/libs/log/handler/debug_only_test.go +++ b/libs/log/handler/debug_only_test.go @@ -2,7 +2,6 @@ package handler import ( "bytes" - "context" "log/slog" "testing" @@ -22,9 +21,9 @@ func TestDebugOnlyHandlerLogsWarningsAtDebug(t *testing.T) { func TestDebugOnlyHandlerEnabledClampsLevel(t *testing.T) { debugBase := slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: log.LevelDebug}) - assert.True(t, DebugOnlyHandler{Handler: debugBase}.Enabled(context.Background(), slog.LevelWarn)) + assert.True(t, DebugOnlyHandler{Handler: debugBase}.Enabled(t.Context(), slog.LevelWarn)) // A warning is still filtered when the base handler's level is above Debug. errorBase := slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelError}) - assert.False(t, DebugOnlyHandler{Handler: errorBase}.Enabled(context.Background(), slog.LevelWarn)) + assert.False(t, DebugOnlyHandler{Handler: errorBase}.Enabled(t.Context(), slog.LevelWarn)) } From 547699e0ebf5115f945acf150ea1d9767c61f871 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 19:44:10 +0000 Subject: [PATCH 36/43] Adapt card 10a to the --names refusal from card 05 The refusal uses the MCP name-conflict category from the telemetry protos, as the other name conflicts do. Co-authored-by: Isaac --- cmd/aitools/mcp_remove.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index ec344724c02..7c92789d1ae 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -258,7 +258,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if !jsonMode { printRemoveResults(ctx, out) } - return failure(categoryNameConflict, "nothing was removed: named MCP servers are left in place: %s", refusedNames(refused)) + return failure(protos.AitoolsErrorCategoryMCPNameConflict, "nothing was removed: named MCP servers are left in place: %s", refusedNames(refused)) } } From 1b5371e5facd9725af5d14af70d7442a132ccd70 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 19:28:19 +0000 Subject: [PATCH 37/43] aitools mcp add --login: log a failed sign-in with a PENDING state when its URL fails When the sign-in URL cannot be built, the SIGN_IN event carried no sign-in state. It now logs the sign-in as FAILED with the PENDING state that a failed sign-in has. The install record still keeps no state. Co-authored-by: Isaac --- cmd/aitools/mcp_add.go | 5 ++++- cmd/aitools/mcp_telemetry_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index 563ea34439d..3cf311c46b4 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -1274,12 +1274,15 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp } var state string var alreadySignedIn bool + // A sign-in whose URL cannot be built is logged as a failed sign-in. The install record keeps no state for it. + eventState := signInStateFailed url, err := mcp.SignInURL(w.Config.Host, plans[i].service.FullName, req.workspaceID) if err == nil { state, alreadySignedIn, _, err = signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) + eventState = state } server.SignIn = &addSignInOutput{State: state} - logMcpEvent(ctx, signInEventOf(server.Service, state, alreadySignedIn, err)) + logMcpEvent(ctx, signInEventOf(server.Service, eventState, alreadySignedIn, err)) if err != nil { server.SignIn.ErrorCategory = errorCategory(err) server.SignIn.Error = err.Error() diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 037c7b58c3d..617c4a02f28 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -5,17 +5,20 @@ import ( "context" "encoding/json" "errors" + "io" "log/slog" "maps" "path/filepath" "slices" "testing" + "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -305,6 +308,28 @@ func TestSignInEventOf(t *testing.T) { }, signInEventOf("system.ai.github", signInStateActive, true, nil)) } +// TestSignInInstalledURLFailureLogsFailedSignIn checks that an add --login whose sign-in URL cannot be built logs the +// sign-in as FAILED with the PENDING state of a failed sign-in, and records no state in the install record. +func TestSignInInstalledURLFailureLogsFailedSignIn(t *testing.T) { + withHome(t) + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + w := &databricks.WorkspaceClient{Config: &config.Config{Host: "https://adb-1.test"}} + servers := []addServerOutput{{ + Service: "system.ai.github", SignInRequired: true, + Agents: []addAgentOutput{{Name: agents.NameClaudeCode, Status: statusInstalled}}, + }} + plans := []addPlan{{service: mcp.Service{FullName: "system.ai.github", Leaf: "github"}}} + + require.Error(t, signInInstalled(ctx, io.Discard, w, addRequest{}, plans, servers)) + assert.Empty(t, servers[0].SignIn.State) + events := telemetry.BufferedEvents(ctx) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) + assert.Equal(t, protos.AitoolsMcpSignInStatePending, events[0].AitoolsMcpEvent.Servers[0].SignInState) +} + // TestMcpEventOmitsUserContent checks that the event carries enums and counts only: a custom service's name must not // appear anywhere in the payload. func TestMcpEventOmitsUserContent(t *testing.T) { From 4b31731e56a3fc5c233bf60b054974f56562ef3d Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 19:29:44 +0000 Subject: [PATCH 38/43] aitools mcp telemetry: an entry already gone is CHANGED, not SKIPPED The proto mirror's comment and the mcp-remove telemetry test title said that an entry already gone from the agent config is SKIPPED. Remove reports it CHANGED, since the record is dropped. Fix the comment and the title. The outcome values are unchanged. Co-authored-by: Isaac --- acceptance/cmd/aitools/telemetry/mcp-remove/output.txt | 2 +- acceptance/cmd/aitools/telemetry/mcp-remove/script | 2 +- libs/telemetry/protos/aitools_mcp.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt index 1d732f75094..9afd121def2 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -63,7 +63,7 @@ "bound_profile_count": 1 } -=== An entry already gone from the agent config is SKIPPED, and the state record goes +=== An entry already gone from the agent config is CHANGED, and the state record goes >>> [CLI] aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json { diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index ef98d35f619..f647d0753dc 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -21,7 +21,7 @@ jq '.mcpServers["genie_one_mcp-logfood"].command = "/usr/bin/other"' home/.claud trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null mcp_events -title "An entry already gone from the agent config is SKIPPED, and the state record goes" +title "An entry already gone from the agent config is CHANGED, and the state record goes" jq 'del(.mcpServers["genie_one_mcp-logfood"])' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null mcp_events diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index aae953d4ede..b153f791dc5 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -68,8 +68,8 @@ const ( // operation writes agent config. Unset fields are omitted: an agent is not set // for sign-in, and an outcome is not set for proxy events. An error category is // set on a FAILED target and on SIGN_IN_PROMPTED, where it is SIGN_IN_REQUIRED. A -// SKIPPED target carries none: remove skips an entry it left in place, or one -// already gone, without a failure. +// SKIPPED target carries none: remove skips an entry it left in place, without +// a failure. type AitoolsMcpServerResult struct { Service AitoolsMcpService `json:"service"` Agent AitoolsAgentType `json:"agent,omitempty"` From 44b16f3c0cf47b203981f79773b68e637f8805b4 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 19:33:42 +0000 Subject: [PATCH 39/43] aitools mcp remove telemetry: group the upload by profile and config file The upload config of remove came from DATABRICKS_CONFIG_FILE alone, so a server pinned to another config file lost its upload. The drift check also ignored config files, so a same-named profile in another file suppressed a valid upload. Both now group the selected servers by profile and resolved config file, as list does. The upload uses that file, and the drift check reads only the servers of the group. Co-authored-by: Isaac --- cmd/aitools/mcp_remove.go | 23 ++++++----------- cmd/aitools/mcp_remove_test.go | 45 ++++++++++++++++++++++++++++++++++ cmd/aitools/mcp_telemetry.go | 32 +++++++++++++----------- 3 files changed, 71 insertions(+), 29 deletions(-) diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 7c92789d1ae..a48d2d90027 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -12,9 +12,7 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/aitools/mcp/agentconfig" - "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" @@ -40,9 +38,9 @@ type removeOutput struct { results []protos.AitoolsMcpServerResult // stateBefore is the state as it was read before the removal. It is not in the JSON. stateBefore *mcp.State - // profile is the profile that the selected servers are bound to, when they all are bound to one. It is "" otherwise. - // It is not in the JSON. - profile string + // profile is the profile and config file that the selected servers are bound to, when they all are bound to one. It is + // the zero key otherwise. It is not in the JSON. + profile profileKey } type removeRemoved struct { @@ -155,21 +153,16 @@ func logRemoveEvent(ctx context.Context, out *removeOutput) { }) { return } - if out.profile == "" { - return - } - configFile, err := mcp.ResolveConfigFile(ctx, env.Get(ctx, mcpconfig.ConfigFileEnv)) - if err != nil { - log.Debugf(ctx, "telemetry: cannot resolve the config file to upload with: %s", err) + if out.profile.profile == "" { return } // The upload sends the profile's token to the profile's host, so a profile whose recorded servers no longer match it // uploads nothing. The records are the ones read before the removal. - if bindingDrifted(ctx, out.stateBefore, out.profile, configFile) { - log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", out.profile) + if bindingDrifted(ctx, out.stateBefore, out.profile) { + log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", out.profile.profile) return } - setUploadConfig(ctx, out.profile, configFile) + setUploadConfig(ctx, out.profile.profile, out.profile.path) } // runMcpRemove removes the chosen servers from the agent configs and records the result. Results are printed @@ -246,7 +239,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // --agents chooses the servers installed for those agents, so a server with no agent left is not chosen. selected = slices.DeleteFunc(selected, func(e mcp.Entry) bool { return len(e.Agents) == 0 }) } - out.profile = sharedProfile(selected) + out.profile = sharedProfile(selected, mcp.CheckEntries(ctx, &mcp.State{Servers: selected})) // Each agent config file is read once per command, and each server's config file is resolved once. files := newAgentFiles() diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 2238e30edf2..9ebb2107834 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -1099,6 +1099,51 @@ func TestMcpRemoveUploadIgnoresEnvironmentToken(t *testing.T) { assert.Equal(t, "t", cfg.Token) } +// TestMcpRemoveUploadUsesConfigFileOfServer checks that a server pinned to a config file uploads with the profile of that +// file, not with a profile of the same name in the file DATABRICKS_CONFIG_FILE names. +func TestMcpRemoveUploadUsesConfigFileOfServer(t *testing.T) { + home := withHome(t) + pinned := filepath.Join(home, "pinned.databrickscfg") + require.NoError(t, os.WriteFile(pinned, []byte("[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = pinned-token\n"), 0o600)) + other := filepath.Join(home, "other.databrickscfg") + require.NoError(t, os.WriteFile(other, []byte("[dogfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = other-token\n"), 0o600)) + claude := agents.ByName("claude-code") + t.Setenv("DATABRICKS_CONFIG_FILE", pinned) + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + t.Setenv("DATABRICKS_CONFIG_FILE", other) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "pinned-token", cfg.Token) +} + +// TestMcpRemoveUploadIgnoresSameNamedProfileInOtherFile checks that a profile of the same name in another config file does +// not stop the upload for the server that matches its own file. +func TestMcpRemoveUploadIgnoresSameNamedProfileInOtherFile(t *testing.T) { + home := withHome(t) + first := filepath.Join(home, "first.databrickscfg") + require.NoError(t, os.WriteFile(first, []byte("[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = first-token\n"), 0o600)) + second := filepath.Join(home, "second.databrickscfg") + require.NoError(t, os.WriteFile(second, []byte("[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = second-token\n"), 0o600)) + claude := agents.ByName("claude-code") + t.Setenv("DATABRICKS_CONFIG_FILE", first) + firstEntry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + t.Setenv("DATABRICKS_CONFIG_FILE", second) + secondEntry := seedInstall(t, claude, "dbsql-logfood", "system.ai.dbsql", "logfood", "https://adb-2.test") + saveState(t, firstEntry, secondEntry) + t.Setenv("DATABRICKS_CONFIG_FILE", first) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "first-token", cfg.Token) +} + // TestMcpRemoveOfflineSetsUploadWithoutWarning checks that a remove that cannot reach the workspace warns nothing: the // upload's config is resolved, and its host metadata lookup fails quietly, so there is no warning to report. func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index f8fb458facb..a86f6e9fbe9 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -268,30 +268,34 @@ func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn } } -// bindingDrifted reports whether a server recorded for profileName in state no longer matches the profile in configFile: -// its host or workspace changed, or its config file moved. The upload would send the profile's token to the host the -// profile names now, which the server was not set up against. It reads the config files only. -func bindingDrifted(ctx context.Context, state *mcp.State, profileName, configFile string) bool { - for _, e := range state.Servers { - if e.Profile != profileName { +// bindingDrifted reports whether a server recorded for the profile of key no longer matches that profile in its config +// file: its host or workspace changed. The upload would send the profile's token to the host the profile names now, which +// the server was not set up against. Only the servers grouped under key are checked, so a profile of the same name in +// another config file does not count. It reads the config files only. +func bindingDrifted(ctx context.Context, state *mcp.State, key profileKey) bool { + checks := mcp.CheckEntries(ctx, state) + for i, e := range state.Servers { + if profileGroup(e, checks[i]) != key { continue } - if _, err := mcp.CheckBinding(ctx, state, e.Service, profileName, configFile); err != nil { + if _, err := mcp.CheckBinding(ctx, state, e.Service, key.profile, key.path); err != nil { return true } } return false } -// sharedProfile returns the profile that all entries are bound to, or "" when they are bound to none or to several. -func sharedProfile(entries []mcp.Entry) string { +// sharedProfile returns the profile and config file that all entries are bound to, grouped as list groups them, or the +// zero key when they are bound to none or to several. checks are the checks of entries, in the same order. +func sharedProfile(entries []mcp.Entry, checks []mcp.EntryCheck) profileKey { if len(entries) == 0 { - return "" + return profileKey{} } - for _, e := range entries[1:] { - if e.Profile != entries[0].Profile { - return "" + first := profileGroup(entries[0], checks[0]) + for i, e := range entries[1:] { + if profileGroup(e, checks[i+1]) != first { + return profileKey{} } } - return entries[0].Profile + return first } From a79f3d769b1d534a88c58efe11097c854785f731 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 19:37:03 +0000 Subject: [PATCH 40/43] aitools mcp serve: set the upload config at start, so the upload uses the bound profile serve resolved the upload config after the session, so the exit upload used whatever the config file held then. It now resolves the config once the serving start is logged, so the upload uses the profile as it was bound. A run rejected at start still sets no config. Co-authored-by: Isaac --- cmd/aitools/mcp_serve.go | 7 +++---- cmd/aitools/mcp_serve_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index dc7368c7be0..ce4e8f691d7 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -265,8 +265,10 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { // serveChallenge fails only on an auth failure (see lookupAtStart), which exits serveBindStatus. return serveFailure(cmd, err, serveBindStatus) } - // Only a serving start is logged. + // Only a serving start is logged. The upload authenticates with the profile as it was bound here, with a config of its + // own that never refreshes a token, since the session's config refreshes its token. logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service)) + setUploadConfig(cmd.Context(), profileName, cfg.ConfigFile) err = serveProxy(ctx, cmd, refresh, proxy.Options{ In: cmd.InOrStdin(), @@ -283,9 +285,6 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { }, Challenge: challenge, }) - // The upload authenticates with the profile at exit, with a config of its own that never refreshes a token, since the - // session's config refreshes its token. A refused upload (proxyResult) still resolves this config, and Upload ignores it. - setUploadConfig(cmd.Context(), profileName, cfg.ConfigFile) return err } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index b4db7b31d46..ca948a89bf2 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -594,6 +594,38 @@ func TestMcpServeRejectedStartLogsNothing(t *testing.T) { assert.False(t, uploads) } +// TestMcpServeUploadUsesProfileBoundAtStart checks that the exit upload of a serving run authenticates with the profile as +// it was bound at start, so a token that the config file gets during the session does not change the upload. +func TestMcpServeUploadUsesProfileBoundAtStart(t *testing.T) { + home := withHome(t) + server := newEchoMCPServer(t) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nworkspace_id = 222\nauth_type = pat\ntoken = dapi-bound\n") + + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + var stdout lockedBuffer + parent.SetOut(&stdout) + in, stdin := io.Pipe() + parent.SetIn(in) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := env.Set(telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + done := make(chan error, 1) + go func() { done <- parent.ExecuteContext(ctx) }() + + _, err := io.WriteString(stdin, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}`+"\n") + require.NoError(t, err) + require.Eventually(t, func() bool { return strings.Contains(stdout.String(), `"id":1`) }, 10*time.Second, time.Millisecond) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nworkspace_id = 222\nauth_type = pat\ntoken = dapi-rotated\n") + require.NoError(t, stdin.Close()) + require.NoError(t, <-done) + + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + cfg, uploads := telemetry.UploadConfig(serve.Context()) + require.True(t, uploads) + assert.Equal(t, "dapi-bound", cfg.Token) +} + // TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it // exits 2 like the other binding failures. func TestMcpServeStateReadFailureExitsTwo(t *testing.T) { From 62401bc3fa4d0b76fce655258b4be022dbc5312d Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 19:52:30 +0000 Subject: [PATCH 41/43] Adapt card 10a to the remove changes from card 05 A record for an unknown agent is now kept, so the record-only test uses pi, a known agent without MCP support. A --names run that names a kept entry now refuses, so the SKIPPED telemetry case removes by --profile. Co-authored-by: Isaac --- acceptance/cmd/aitools/telemetry/mcp-remove/output.txt | 2 +- acceptance/cmd/aitools/telemetry/mcp-remove/script | 2 +- cmd/aitools/mcp_remove_test.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt index 9afd121def2..0910ae20685 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -49,7 +49,7 @@ === An entry changed outside this CLI is left in place: SKIPPED ->>> [CLI] aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json +>>> [CLI] aitools mcp remove --profile logfood --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "REMOVE", diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index f647d0753dc..ed731367c47 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -18,7 +18,7 @@ mcp_events title "An entry changed outside this CLI is left in place: SKIPPED" jq '.mcpServers["genie_one_mcp-logfood"].command = "/usr/bin/other"' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json -trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +trace $CLI aitools mcp remove --profile logfood --output json > /dev/null mcp_events title "An entry already gone from the agent config is CHANGED, and the state record goes" diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 9ebb2107834..434a8412e30 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -1225,11 +1225,11 @@ func TestMcpRemoveFailedStateReadAfterAnotherEventLogsNothing(t *testing.T) { // FAILED when the state cannot be saved, since nothing was removed. func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { withHome(t) - retired := mcp.Entry{ + recordOnly := mcp.Entry{ Name: "dbsql-logfood", Service: "system.ai.dbsql", Profile: "logfood", Host: "https://adb-1.test", Command: testDatabricksCommand, - Agents: map[string]mcp.AgentRecord{"retired-agent": {Path: "/x/config.json", Scope: "user"}}, + Agents: map[string]mcp.AgentRecord{"pi": {Path: "/x/config.json", Scope: "user"}}, } - saveState(t, retired) + saveState(t, recordOnly) saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } t.Cleanup(func() { saveMCPState = mcp.SaveState }) From 7df6abe8bb6e945983a1006b80598242b572c067 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 21:34:56 +0000 Subject: [PATCH 42/43] aitools mcp: upload telemetry after a rejected credential A run that saw a rejected credential refused its upload, since the upload would present that credential again. The refusal is removed: telemetry.RefuseUpload and its state, the callers in serve and logMcpEvent, and the proxy's Recorder.AuthRejected hook, which only called it. An event that names a rejected credential is logged, and the run uploads like any other, with the profile it bound at start, whose token never refreshes. Serve still sets its upload config at PROXY_STARTED. A start that is rejected logs nothing, so there is nothing to upload whatever config is set. Co-authored-by: Isaac --- cmd/aitools/mcp_serve.go | 4 --- cmd/aitools/mcp_serve_test.go | 6 ++-- cmd/aitools/mcp_telemetry.go | 24 ++++------------ cmd/aitools/mcp_telemetry_test.go | 35 ++++-------------------- libs/aitools/mcp/proxy/challenge.go | 4 --- libs/aitools/mcp/proxy/challenge_test.go | 27 +----------------- libs/telemetry/logger.go | 24 ++-------------- libs/telemetry/logger_test.go | 24 ---------------- 8 files changed, 18 insertions(+), 130 deletions(-) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index ce4e8f691d7..12fe7ea58f4 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -29,7 +29,6 @@ import ( "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/shellquote" - "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" @@ -477,9 +476,6 @@ func serveProxy(ctx context.Context, cmd *cobra.Command, refresh *refreshGuard, // stdin error after it is a consequence. The final line always names the rejection, as the bridge may not have logged it. func proxyResult(cmd *cobra.Command, refresh *refreshGuard, runErr error) error { if refreshErr := refresh.failure(); refreshErr != nil { - // The token was rejected or the binding changed, so the upload at exit would only ask the token endpoint again or - // use the re-pointed profile. - telemetry.RefuseUpload(cmd.Context()) fmt.Fprintf(cmd.ErrOrStderr(), "Error: stopped after %s for profile \"%s\"\n", stopReason(refreshErr), refresh.profile) return exitAlreadyPrinted(refreshErr, serveBindStatus) } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index ca948a89bf2..5527ec1d90c 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -574,7 +574,7 @@ func TestMcpServeFailedStartLogsNothing(t *testing.T) { } // TestMcpServeRejectedStartLogsNothing checks that a start whose startup read is rejected by the profile's credential logs -// no PROXY_STARTED, so the run uploads nothing: the rejected token is not sent again. +// no PROXY_STARTED, so the run has nothing to upload, and sets no upload config. func TestMcpServeRejectedStartLogsNothing(t *testing.T) { home := withHome(t) server := serviceServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) @@ -685,9 +685,9 @@ func TestProxyResultRefreshFailureWinsOverStdinError(t *testing.T) { err := proxyResult(cmd, g, errors.New("read stdin: input/output error")) - // The exit-time upload would ask the rejected token's endpoint again, so the run uploads nothing. + // The run uploads like any other: its upload config is the profile bound at start, which never refreshes a token. _, uploads := telemetry.UploadConfig(cmd.Context()) - assert.False(t, uploads) + assert.True(t, uploads) assert.ErrorIs(t, err, root.ErrAlreadyPrinted) assert.Equal(t, serveBindStatus, root.ExitStatus(err)) assert.ErrorIs(t, err, cause) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index a86f6e9fbe9..8a445f15184 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -2,7 +2,6 @@ package aitools import ( "context" - "slices" "strings" "github.com/databricks/cli/libs/aitools/agents" @@ -76,20 +75,13 @@ func proxyAgentType(flag string) protos.AitoolsAgentType { } // logMcpEvent buffers an MCP event, and reports whether it did; cmd/root uploads it at exit. An event without server -// results is not logged: a run that stopped before any server was reached has nothing to report. An event that names a -// rejected credential is not logged either, and the run refuses its upload, since the upload would present that -// credential again. The bound profile count is read from the state file as it is after the operation. A state file that -// cannot be read drops the event; telemetry never fails a command. +// results is not logged: a run that stopped before any server was reached has nothing to report. The bound profile count +// is read from the state file as it is after the operation. A state file that cannot be read drops the event; telemetry +// never fails a command. func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) bool { if len(event.Servers) == 0 { return false } - if slices.ContainsFunc(event.Servers, func(r protos.AitoolsMcpServerResult) bool { - return r.ErrorCategory == protos.AitoolsErrorCategoryAuthFailed - }) { - telemetry.RefuseUpload(ctx) - return false - } state, err := mcp.LoadState(ctx) if err != nil { log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) @@ -222,9 +214,8 @@ func signOutServerResult(service, state string, err error) protos.AitoolsMcpServ } // proxyStartedEvent is the PROXY_STARTED event of serve, logged once the startup checks have passed and the proxy serves. -// A start that fails its checks logs nothing. A binding that failed has no profile the upload may authenticate with -// (PROFILE_NOT_FOUND and PROFILE_HOST_CHANGED are never sent), and a rejected credential would only repeat its rejected -// request, so none of them uploads. +// A start that fails its checks, whether at the binding or on a rejected credential, logs nothing, so its run has nothing +// to upload. func proxyStartedEvent(agent protos.AitoolsAgentType, service string) protos.AitoolsMcpEvent { return protos.AitoolsMcpEvent{ CallingAgent: agent, @@ -245,11 +236,6 @@ func (t mcpProxyTelemetry) SignInPrompted(ctx context.Context, signedIn bool) { logMcpEvent(ctx, signInPromptedEvent(t.agent, t.service, signedIn)) } -// AuthRejected refuses the run's upload, since the upload would present the rejected credential again. -func (t mcpProxyTelemetry) AuthRejected(ctx context.Context) { - telemetry.RefuseUpload(ctx) -} - // signInPromptedEvent is the SIGN_IN_PROMPTED event of serve: the sign-in is required, and the // sign-in state is ACTIVE when the wait ended with the credential, PENDING otherwise. func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn bool) protos.AitoolsMcpEvent { diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 617c4a02f28..e6e7ec7f3fc 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -193,14 +193,13 @@ func TestProxyStartedEvent(t *testing.T) { }, proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp")) } -// TestLogMcpEventDropsRejectedCredential checks that an event that names a rejected credential is not logged, and that the -// run then uploads nothing: its upload would authenticate with that credential again. The events of the run that were -// logged before stay buffered, and a later event of the run is logged, but not uploaded. -func TestLogMcpEventDropsRejectedCredential(t *testing.T) { +// TestLogMcpEventLogsRejectedCredential checks that an event that names a rejected credential is logged, and that the run +// uploads with it like any other run: the upload authenticates with the profile the run bound, which never refreshes. +func TestLogMcpEventLogsRejectedCredential(t *testing.T) { withHome(t) ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") telemetry.SetUploadConfig(ctx, &config.Config{Profile: "logfood", Host: "https://adb-1.test"}) - assert.False(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ + assert.True(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ Operation: protos.AitoolsMcpOperationSignIn, Servers: []protos.AitoolsMcpServerResult{{ Service: protos.AitoolsMcpServiceGitHub, @@ -208,21 +207,9 @@ func TestLogMcpEventDropsRejectedCredential(t *testing.T) { ErrorCategory: protos.AitoolsErrorCategoryAuthFailed, }}, })) - assert.Empty(t, telemetry.BufferedEvents(ctx)) - _, uploads := telemetry.UploadConfig(ctx) - assert.False(t, uploads) - - assert.True(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ - Operation: protos.AitoolsMcpOperationAdd, - Servers: []protos.AitoolsMcpServerResult{{ - Service: protos.AitoolsMcpServiceGitHub, - Outcome: protos.AitoolsTargetOutcomeChanged, - SignInState: protos.AitoolsMcpSignInStatePending, - }}, - })) assert.Len(t, telemetry.BufferedEvents(ctx), 1) - _, uploads = telemetry.UploadConfig(ctx) - assert.False(t, uploads, "the add event is logged, and it is not uploaded with the rejected run") + _, uploads := telemetry.UploadConfig(ctx) + assert.True(t, uploads) } // TestSetUploadConfigLogsNoWarningForFailedHostMetadata checks that an upload whose host metadata lookup fails warns @@ -252,16 +239,6 @@ func TestSetUploadConfigLogsNoWarningForFailedHostMetadata(t *testing.T) { assert.Empty(t, warnings.String()) } -// TestMcpProxyAuthRejectedRefusesUpload checks that a 401 the proxy answers refuses the run's upload, even though the -// serve command sets its upload config at exit. -func TestMcpProxyAuthRejectedRefusesUpload(t *testing.T) { - ctx := telemetry.WithNewLogger(t.Context()) - telemetry.SetUploadConfig(ctx, &config.Config{Profile: "logfood", Host: "https://adb-1.test"}) - mcpProxyTelemetry{}.AuthRejected(ctx) - _, uploads := telemetry.UploadConfig(ctx) - assert.False(t, uploads) -} - func TestSignInPromptedEvent(t *testing.T) { for signedIn, state := range map[bool]protos.AitoolsMcpSignInState{ true: protos.AitoolsMcpSignInStateActive, diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index 676b057c1ae..5f54d0f6090 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -57,9 +57,6 @@ type Recorder interface { // earlier cancelled call opened, which the user may still have open. signedIn reports whether the credential became // ACTIVE. A retry after a cancelled call reports its own outcome. SignInPrompted(ctx context.Context, signedIn bool) - // AuthRejected runs when the service answers 401 without asking for a sign-in: the Databricks credential was - // rejected, and the answer is the auth failure. - AuthRejected(ctx context.Context) } // Challenge turns on sign-in handling for a service that needs a provider sign-in. @@ -180,7 +177,6 @@ func (h *signInHooks) Result(ctx context.Context, request []byte, status int, me } challenged := h.challenged(&reply) if status == http.StatusUnauthorized && !challenged { - h.c.Recorder.AuthRejected(ctx) return [][]byte{errorResponse(req.ID, codeUpstream, h.authFailure())}, nil } if req.Method != methodToolsList && req.Method != methodToolsCall { diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 48f3ab8405c..78b8633f485 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -68,12 +68,10 @@ func (s *signInStub) requests() []SignInRequest { // loginCommand is the hint that the command builds for a pinned config file and a profile name with a space. const loginCommand = `DATABRICKS_CONFIG_FILE='/work/cfg/my file.cfg' databricks aitools mcp login system.ai.github --profile 'my profile'` -// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. It counts the -// rejections of the Databricks credential too. +// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. type promptRecorder struct { mu sync.Mutex signedIn []bool - rejected int } func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { @@ -82,12 +80,6 @@ func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { r.signedIn = append(r.signedIn, signedIn) } -func (r *promptRecorder) AuthRejected(_ context.Context) { - r.mu.Lock() - defer r.mu.Unlock() - r.rejected++ -} - func (r *promptRecorder) prompts() []bool { r.mu.Lock() defer r.mu.Unlock() @@ -1030,23 +1022,6 @@ func TestUnauthorizedIsAuthFailureUnlessItAsksForSignIn(t *testing.T) { } } -// TestUnauthorizedRecordsRejectedCredential checks that a 401 without a sign-in challenge is recorded as a rejected -// credential, once per answer, so that the command refuses its telemetry upload. -func TestUnauthorizedRecordsRejectedCredential(t *testing.T) { - srv, _ := newFakeEndpoint(t, func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, "unauthorized", http.StatusUnauthorized) - }) - signIn := &signInStub{reply: func(int) (bool, error) { return false, nil }} - recorder := &promptRecorder{} - challenge := testChallenge(signIn.SignIn) - challenge.Recorder = recorder - b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(challenge, io.Discard) - runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") - - assert.Equal(t, 1, recorder.rejected) -} - // TestRejectedCredentialDoesNotAskForLogout checks a rejected Databricks token after "Already signed in": the credential reads // as signed in, and the service rejects the token with a 401. The 401 is an auth failure, not a challenge, so the next // authenticate runs the sign-in as usual, and the logout answer is not given. diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index 32a228cb404..20070c0bd30 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -48,29 +48,13 @@ func SetUploadConfig(ctx context.Context, cfg *config.Config) { } } -// RefuseUpload makes Upload send nothing for the run, whatever config is set. A run calls it once a credential was -// rejected, since the upload would present that credential again. Like Log, it is a no-op without a logger. -func RefuseUpload(ctx context.Context) { - if l, ok := loggerFromContext(ctx); ok { - l.mu.Lock() - defer l.mu.Unlock() - l.refused = true - } -} - // UploadConfig returns the config that Upload authenticates with: the one SetUploadConfig set, else the one -// cmdctx.ConfigUsed holds. It returns false when the run refused its upload or has no config. +// cmdctx.ConfigUsed holds. It returns false when the run has no config. func UploadConfig(ctx context.Context) (*config.Config, bool) { l, ok := loggerFromContext(ctx) if !ok { return nil, false } - l.mu.Lock() - refused := l.refused - l.mu.Unlock() - if refused { - return nil, false - } if l.upload != nil { return l.upload, true } @@ -94,15 +78,13 @@ func Log(ctx context.Context, event protos.DatabricksCliLog) { } type logger struct { - // mu guards logs and refused, which the goroutines of a command's MCP proxy write concurrently. + // mu guards logs, which the goroutines of a command's MCP proxy write concurrently. mu sync.Mutex logs []protos.FrontendLog stateLineageHash string // upload is the config Upload authenticates with, as SetUploadConfig set it. Nil leaves cmdctx.ConfigUsed to it. upload *config.Config - // refused means the run refused its upload (RefuseUpload), so Upload sends nothing. - refused bool // running counts the goroutines that Upload started and stopped waiting // for, which may still be using the config (see within). @@ -181,7 +163,7 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { cfg, ok := UploadConfig(ctx) if !ok { - log.Debugf(ctx, "no auth config available, or the run refused its telemetry upload; skipping telemetry upload") + log.Debugf(ctx, "no auth config available; skipping telemetry upload") return nil } diff --git a/libs/telemetry/logger_test.go b/libs/telemetry/logger_test.go index a6a01c9a72e..7eb218094b7 100644 --- a/libs/telemetry/logger_test.go +++ b/libs/telemetry/logger_test.go @@ -497,34 +497,10 @@ func TestUploadUsesConfigSetByCommand(t *testing.T) { assert.Zero(t, otherCalls) } -// TestUploadRefusedSendsNothing checks that a run which refused its upload sends nothing, even with a config set by the -// command, and that a config set after the refusal is not used. -func TestUploadRefusedSendsNothing(t *testing.T) { - server := testserver.New(t) - t.Cleanup(server.Close) - var calls int - server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { - calls++ - return ResponseBody{NumProtoSuccess: 1} - }) - - ctx := WithNewLogger(t.Context()) - Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) - SetUploadConfig(ctx, &config.Config{Host: server.URL, Token: "token"}) - RefuseUpload(ctx) - SetUploadConfig(ctx, &config.Config{Host: server.URL, Token: "token"}) - - _, ok := UploadConfig(ctx) - assert.False(t, ok) - require.NoError(t, Upload(ctx, protos.ExecutionContext{})) - assert.Zero(t, calls) -} - // TestUploadConfigWithoutLogger checks that the upload slot is a no-op without a logger, as Log is. func TestUploadConfigWithoutLogger(t *testing.T) { ctx := t.Context() SetUploadConfig(ctx, &config.Config{Host: "https://ws.test"}) - RefuseUpload(ctx) _, ok := UploadConfig(ctx) assert.False(t, ok) } From 9910702619717b07b429ee79bad51fb808bd98f0 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Tue, 17 Nov 2026 22:12:12 +0000 Subject: [PATCH 43/43] aitools mcp serve: drop the exit upload when the profile was re-pointed The upload authenticates with the profile, and its token comes from the token cache, which is keyed by profile name alone. A profile re-pointed at another host during the session, by a new login with --host, would then send that host's token to the host serve bound. Serve now checks the binding again when the proxy ends, as remove does, and drops the upload config when it no longer holds. Co-authored-by: Isaac --- cmd/aitools/mcp_serve.go | 15 ++++++++++++++- cmd/aitools/mcp_serve_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 12fe7ea58f4..f21e2bda2f5 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -29,6 +29,7 @@ import ( "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" @@ -238,7 +239,8 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { defer stop() // The bridge and the sign-in client share the guard, so a refresh failure in either one stops the proxy. // Each token fetch re-checks the binding first; see rebindingAuthenticate. - authenticate := rebindingAuthenticate(cmd.Context(), bindingFor(state, entry, f.service, cfg), f.service, profileName, cfg.ConfigFile, cfg.Authenticate) + binding := bindingFor(state, entry, f.service, cfg) + authenticate := rebindingAuthenticate(cmd.Context(), binding, f.service, profileName, cfg.ConfigFile, cfg.Authenticate) refresh := &refreshGuard{authenticate: authenticate, profile: profileName, pinnedFile: pinnedFile, cancel: stop} // The sign-in client is built only when serveChallenge reads the service or signs in. A server whose record says it @@ -284,9 +286,20 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { }, Challenge: challenge, }) + dropUploadIfRebound(cmd.Context(), binding, f.service, profileName, cfg.ConfigFile) return err } +// dropUploadIfRebound drops the exit upload when the profile no longer names the host that serve bound. The upload's +// token comes from the token cache, which is keyed by profile name alone, so after a re-point it would be the new host's +// token, sent to the bound host. remove drops its upload for a drifted binding the same way. +func dropUploadIfRebound(ctx context.Context, state *mcp.State, service, profileName, configFile string) { + if _, err := mcp.CheckBinding(ctx, state, service, profileName, configFile); err != nil { + log.Debugf(ctx, "telemetry: profile %s no longer matches the binding serve started with; not uploading telemetry", profileName) + telemetry.SetUploadConfig(ctx, nil) + } +} + // bindingFor returns the state that the per-token binding check reads. With an install record, that is state. Without // one, nothing records the host the profile was bound to, so the entry is made from cfg, which Bind resolved at start. func bindingFor(state *mcp.State, entry *mcp.Entry, service string, cfg *config.Config) *mcp.State { diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index 5527ec1d90c..65c4670946c 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -626,6 +626,38 @@ func TestMcpServeUploadUsesProfileBoundAtStart(t *testing.T) { assert.Equal(t, "dapi-bound", cfg.Token) } +// TestMcpServeDropsUploadAfterRepoint checks that a profile re-pointed at another host during the session leaves serve +// with no upload config at exit, even when no request ran after the re-point. The upload's token comes from the token +// cache by profile name, so it would be the new host's token, sent to the bound host. +func TestMcpServeDropsUploadAfterRepoint(t *testing.T) { + home := withHome(t) + server := newEchoMCPServer(t) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nworkspace_id = 222\nauth_type = pat\ntoken = dapi-bound\n") + + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + var stdout lockedBuffer + parent.SetOut(&stdout) + in, stdin := io.Pipe() + parent.SetIn(in) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := env.Set(telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + done := make(chan error, 1) + go func() { done <- parent.ExecuteContext(ctx) }() + + _, err := io.WriteString(stdin, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}`+"\n") + require.NoError(t, err) + require.Eventually(t, func() bool { return strings.Contains(stdout.String(), `"id":1`) }, 10*time.Second, time.Millisecond) + writeHomeConfig(t, home, "[logfood]\nhost = https://other.test\nworkspace_id = 222\nauth_type = pat\ntoken = dapi-other\n") + require.NoError(t, stdin.Close()) + require.NoError(t, <-done) + + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + _, uploads := telemetry.UploadConfig(serve.Context()) + assert.False(t, uploads) +} + // TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it // exits 2 like the other binding failures. func TestMcpServeStateReadFailureExitsTwo(t *testing.T) {