diff --git a/apps/ai/src/chat/do-runtime.ts b/apps/ai/src/chat/do-runtime.ts index 77c56944e6..1c85b74640 100644 --- a/apps/ai/src/chat/do-runtime.ts +++ b/apps/ai/src/chat/do-runtime.ts @@ -7,7 +7,7 @@ import { type ChatSessionNamespace, chatSessionsLayerIfBound } from "@maple/back import { envPorts } from "@maple/backend/platform/env-ports" import { runtimeTelemetry } from "@maple/infra/worker-telemetry" import { Effect, Layer, Schema } from "effect" -import { MCP_ANTICIPATED_ERROR_IDENTIFIERS } from "../mcp/expected-failures" +import { MCP_ANTICIPATED_ERROR_IDENTIFIERS, isAnticipatedMcpFailure } from "../mcp/expected-failures" import type { AiSessionPortsLayer } from "../worker/bindings" import { provideSessionPorts } from "./session-ports" @@ -46,6 +46,7 @@ const ENGINE_BOOKKEEPING_SPANS = [ export const chatTelemetry = runtimeTelemetry({ serviceName: "maple-chat", anticipatedErrorIdentifiers: MCP_ANTICIPATED_ERROR_IDENTIFIERS, + isAnticipatedError: isAnticipatedMcpFailure, dropSpanNames: ENGINE_BOOKKEEPING_SPANS, }) diff --git a/apps/ai/src/mcp/expected-failures.test.ts b/apps/ai/src/mcp/expected-failures.test.ts new file mode 100644 index 0000000000..d70fd533ba --- /dev/null +++ b/apps/ai/src/mcp/expected-failures.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest" +import { Duration, String as Str } from "effect" +import { IntegrationsPersistenceError, IntegrationsUpstreamError } from "@maple/domain/http" +import { SandboxRunCheckoutFailed, SandboxRunCheckoutPending } from "@maple/domain/sandbox" +import { + SandboxExitError, + SandboxImplementation, + SandboxOutputLimitError, + SandboxSpawnError, + SandboxTimeoutError, + SandboxUnsupportedRequestError, + type SandboxError, +} from "@maple/backend/services/sandbox/contract" +import { VcsSourceRefNotFoundError } from "@maple/backend/services/integrations/vcs/VcsSourceService" +import { isAnticipatedMcpFailure } from "./expected-failures" +import { fromSandboxError } from "./lib/source-errors" + +const implementation = new SandboxImplementation({ isolation: "isolated", identity: "test" }) +const spawn = (cause: unknown): SandboxError => + new SandboxSpawnError({ implementation, command: "git", message: "never started", cause }) + +const SANDBOX_FAILURES: ReadonlyArray = [ + [ + "an invented SHA", + spawn( + new VcsSourceRefNotFoundError({ repository: "o/r", ref: Str.repeat(40)("a"), message: "no ref" }), + ), + ], + ["a clone still running", spawn(new SandboxRunCheckoutPending({ message: "cloning" }))], + [ + "a clone that failed", + spawn(new SandboxRunCheckoutFailed({ message: "Cloning o/r failed (exit 128)" })), + ], + [ + "a GitHub rate limit on the checkout lookup", + spawn( + new IntegrationsUpstreamError({ message: "GitHub rate limited (retry after 60s)", status: 403 }), + ), + ], + [ + "a GitHub outage on the checkout lookup", + spawn(new IntegrationsUpstreamError({ message: "Get commit failed: 502", status: 502 })), + ], + ["a lost database connection", spawn(new IntegrationsPersistenceError({ message: "Connection closed" }))], + ["a timeout", new SandboxTimeoutError({ implementation, maxWallTime: Duration.seconds(30) })], + [ + "too much output", + new SandboxOutputLimitError({ implementation, stream: "stdout", limit: 1024, observed: 2048 }), + ], + [ + "a bad mount", + new SandboxUnsupportedRequestError({ + implementation, + feature: "mounts", + message: "bad mount source", + }), + ], + [ + "no sandbox runtime", + new SandboxUnsupportedRequestError({ implementation, feature: "runtime", message: "no runtime" }), + ], + ["a command that exited", new SandboxExitError({ implementation, exitCode: 2, message: "exit 2" })], +] + +describe("isAnticipatedMcpFailure", () => { + // The span and the model must agree: what the model is told to fix or retry is not a server error. + // A missing capability (no sandbox runtime) is told to the model too, but stays an Error span. + it.each(SANDBOX_FAILURES)("agrees with the model-facing mapping for %s", (_, error) => { + const shownToModel = fromSandboxError("sandbox_exec")(error)._tag + expect(isAnticipatedMcpFailure(error)).toBe( + shownToModel === "@maple/mcp/errors/McpInvalidInputError" || + shownToModel === "@maple/mcp/errors/McpNotReadyError", + ) + }) + + it("treats a call to a tool that does not exist as the client's mistake", () => { + const unknownTool = { _tag: "ProtocolError", code: -32602, message: "Tool 'tools/list' not found" } + const internal = { _tag: "ProtocolError", code: -32603, message: "Internal error" } + expect([isAnticipatedMcpFailure(unknownTool), isAnticipatedMcpFailure(internal)]).toEqual([ + true, + false, + ]) + }) +}) diff --git a/apps/ai/src/mcp/expected-failures.ts b/apps/ai/src/mcp/expected-failures.ts index 23c36f64a0..bcd906c82d 100644 --- a/apps/ai/src/mcp/expected-failures.ts +++ b/apps/ai/src/mcp/expected-failures.ts @@ -1,4 +1,4 @@ -import { Effect, Record as Rec, String as Str } from "effect" +import { Effect, Predicate, Record as Rec, String as Str } from "effect" /** * MCP failures that are expected 4xx client outcomes, not bugs: a bad or absent @@ -37,6 +37,62 @@ const expectedStatusByTag = new Map(Rec.toEntries(MCP_EXPECTED_F /** Spread into `anticipatedErrorIdentifiers` next to `ANTICIPATED_ERROR_IDENTIFIERS`. */ export const MCP_ANTICIPATED_ERROR_IDENTIFIERS: ReadonlyArray = [...expectedStatusByTag.keys()] +/** JSON-RPC codes for a request the client got wrong: an unknown method, or params (a tool name) that name nothing. */ +const CLIENT_PROTOCOL_CODES: ReadonlySet = new Set([-32601, -32602]) + +/** Why a repository command never started, when the reason is the call's ref, repository or path. */ +const CALLER_SPAWN_CAUSES: ReadonlySet = new Set([ + "SandboxRunCheckoutPending", + "@maple/backend/vcs/VcsSourceRefNotFoundError", + "@maple/backend/vcs/VcsSourceRepositoryNotFoundError", + "@maple/backend/vcs/VcsSourceFileNotFoundError", +]) + +// BOUNDARY: a failure value from the tracer, or the cause a sandbox error carries. +/** The GitHub client's wording for a rate limit, which carries its wait in seconds. */ +const GITHUB_RATE_LIMITED = /^GitHub rate limited \(retry after (\d+)s\)/ + +/** The wait a GitHub rate limit asks for, when `message` is one. */ +export const githubRateLimitWait = (message: string): number | undefined => { + const seconds = GITHUB_RATE_LIMITED.exec(message)?.[1] + return seconds === undefined ? undefined : Number(seconds) +} + +const isRateLimitedLookup = (cause: unknown): boolean => + tagOf(cause) === "@maple/http/errors/IntegrationsUpstreamError" && + Predicate.hasProperty(cause, "message") && + typeof cause.message === "string" && + githubRateLimitWait(cause.message) !== undefined + +const tagOf = (value: unknown): string | undefined => + Predicate.hasProperty(value, "_tag") && typeof value._tag === "string" ? value._tag : undefined + +/** + * Failures a tag cannot single out, judged on the value: the SDK's MCP protocol error for a call to + * a tool that does not exist, and repository sandbox failures `fromSandboxError` hands the model as + * a fixable call or one to retry later (a missing capability stays an Error). Both export `Ok`. + * + * BOUNDARY: the tracer hands over a cause's raw failure values. + */ +export const isAnticipatedMcpFailure = (error: unknown): boolean => { + switch (tagOf(error)) { + case "ProtocolError": + return Predicate.hasProperty(error, "code") && CLIENT_PROTOCOL_CODES.has(error.code) + case "@maple/backend/sandbox/SandboxTimeoutError": + case "@maple/backend/sandbox/SandboxOutputLimitError": + return true + case "@maple/backend/sandbox/SandboxUnsupportedRequestError": + return Predicate.hasProperty(error, "feature") && error.feature !== "runtime" + case "@maple/backend/sandbox/SandboxSpawnError": + return ( + Predicate.hasProperty(error, "cause") && + (CALLER_SPAWN_CAUSES.has(tagOf(error.cause)) || isRateLimitedLookup(error.cause)) + ) + default: + return false + } +} + /** * Longest failure message a span or a transcript keeps. A ClickHouse syntax error arrives with the * whole offending query inlined, and on a retry loop it would land once per attempt. diff --git a/apps/ai/src/mcp/lib/source-errors.test.ts b/apps/ai/src/mcp/lib/source-errors.test.ts new file mode 100644 index 0000000000..e1706795e1 --- /dev/null +++ b/apps/ai/src/mcp/lib/source-errors.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest" +import { IntegrationsUpstreamError } from "@maple/domain/http" +import { fromVcsLookupError } from "./source-errors" + +const mapped = (error: IntegrationsUpstreamError) => fromVcsLookupError("search_source_code")(error) + +describe("fromVcsLookupError", () => { + it("asks the model to come back after a GitHub rate limit", () => { + const error = mapped( + new IntegrationsUpstreamError({ message: "GitHub rate limited (retry after 60s)", status: 403 }), + ) + expect(error).toMatchObject({ _tag: "@maple/mcp/errors/McpNotReadyError", retryAfterSeconds: 60 }) + }) + + it("hands a search query GitHub could not parse back to the model to fix", () => { + const error = mapped( + new IntegrationsUpstreamError({ + message: + 'Search code failed: 422 {"message":"ERROR_TYPE_QUERY_PARSING_FATAL unable to parse query!"}', + status: 422, + }), + ) + expect(error._tag).toBe("@maple/mcp/errors/McpInvalidInputError") + }) + + it("keeps a GitHub outage a query failure", () => { + const error = mapped( + new IntegrationsUpstreamError({ + message: "Get pull request failed: 503 Service Unavailable", + status: 503, + }), + ) + expect(error._tag).toBe("@maple/mcp/errors/McpQueryError") + }) +}) diff --git a/apps/ai/src/mcp/lib/source-errors.ts b/apps/ai/src/mcp/lib/source-errors.ts index 7364ffaa85..972dea272e 100644 --- a/apps/ai/src/mcp/lib/source-errors.ts +++ b/apps/ai/src/mcp/lib/source-errors.ts @@ -17,6 +17,7 @@ import { VcsSourceRepositoryNotFoundError, } from "@maple/backend/services/integrations/vcs/VcsSourceService" import { McpInvalidInputError, McpNotReadyError, McpQueryError, McpUnavailableError } from "../tools/types" +import { githubRateLimitWait } from "../expected-failures" export type VcsLookupError = | IntegrationsNotConnectedError @@ -72,10 +73,15 @@ export const fromVcsLookupError = message: "No source repository is connected to this organization.", capability: "source_repository", }) - case "@maple/http/errors/IntegrationsUpstreamError": - return isGithubAppMissing(error) - ? GITHUB_APP_UNAVAILABLE - : new McpQueryError({ message: error.message, pipeName: operation, cause: error }) + case "@maple/http/errors/IntegrationsUpstreamError": { + if (isGithubAppMissing(error)) return GITHUB_APP_UNAVAILABLE + const retryAfter = githubRateLimitWait(error.message) + if (retryAfter !== undefined) + return new McpNotReadyError({ message: error.message, retryAfterSeconds: retryAfter }) + // GitHub could not parse what the call sent (a code search query, most often). + if (error.status === 422) return new McpInvalidInputError({ message: error.message }) + return new McpQueryError({ message: error.message, pipeName: operation, cause: error }) + } case "@maple/http/errors/IntegrationsPersistenceError": return new McpQueryError({ message: error.message, pipeName: operation, cause: error }) } diff --git a/apps/ai/src/platform/Llm.test.ts b/apps/ai/src/platform/Llm.test.ts index 49ece96c6e..851dfa484e 100644 --- a/apps/ai/src/platform/Llm.test.ts +++ b/apps/ai/src/platform/Llm.test.ts @@ -24,10 +24,13 @@ import { FetchHttpClient } from "effect/http" import { assert, describe, it } from "@effect/vitest" import { expect } from "vitest" import { PR_REVIEW_MODELS } from "@maple/domain/http" +import { FindingEmbedder } from "@maple/backend/services/pr-review/FindingEmbedder" import { DEFAULT_DECISION_MODEL, layerDecisionModel, DEFAULT_REVIEW_MODEL, + DEFAULT_EMBEDDING_MODEL, + layerFindingEmbedder, layerLlm, loadLlmSettings, resolveDecisionModel, @@ -518,6 +521,21 @@ describe("EU in-region routing", () => { }) }) +describe("layerFindingEmbedder", () => { + const embedderFor = (settings: LlmSettings) => + Effect.runSync( + Effect.gen(function* () { + const embedder = yield* FindingEmbedder + return Option.map(embedder, (found) => found.model) + }).pipe(Effect.provide(layerFindingEmbedder(settings))), + ) + + it("embeds on OpenRouter in the US and not in the EU, where no provider serves the model", () => { + expect(embedderFor(openRouterEnv)).toEqual(Option.some(DEFAULT_EMBEDDING_MODEL)) + expect(embedderFor({ ...openRouterEnv, MAPLE_REGION: "eu" })).toEqual(Option.none()) + }) +}) + describe("resolveTriageModel — context limits", () => { it("attaches the configured model's window, which upstream leaves unstated", () => { // Providers do not report their context window, so without this table nothing could tell diff --git a/apps/ai/src/platform/Llm.ts b/apps/ai/src/platform/Llm.ts index 6651e97f77..03a0ffbeb7 100644 --- a/apps/ai/src/platform/Llm.ts +++ b/apps/ai/src/platform/Llm.ts @@ -831,11 +831,12 @@ export const resolveDecisionModel = (settings: LlmSettings): string | undefined * The embedder the PR review's feedback filter compares findings with, on OpenRouter whichever * provider the agents run on: Workers AI's binding shim only answers chat. Its `OpenAiClient` is * private to this layer, so it never replaces the Workers AI one `layerLlm` provides. Without an - * OpenRouter key there is no embedder, and the filter is off. + * OpenRouter key there is no embedder, and the filter is off. Nor in the EU: OpenRouter's EU + * endpoint serves no provider for the embedding model ("No endpoints found supporting your data region"). */ export const layerFindingEmbedder = (settings: LlmSettings): Layer.Layer => { const apiKey = readSecret(settings, "OPENROUTER_API_KEY") - if (apiKey === undefined) return FindingEmbedder.none + if (apiKey === undefined || openRouterRegion(settings) === "eu") return FindingEmbedder.none const model = DEFAULT_EMBEDDING_MODEL return Layer.effect(FindingEmbedder)( Effect.map(OpenAiEmbeddingModel.make({ model }), (embeddings) => diff --git a/apps/ai/src/worker.ts b/apps/ai/src/worker.ts index 22868ead02..20c9553c1f 100644 --- a/apps/ai/src/worker.ts +++ b/apps/ai/src/worker.ts @@ -17,7 +17,7 @@ import { ChatSessionObject } from "@maple/backend/platform/chat-sessions" import { ChatSessionLive } from "./chat/ChatSession" import { aiConfiguredEnv } from "./resources/env" import SecurityAuditWorkflow from "./security-audit/SecurityAuditWorkflow" -import { MCP_ANTICIPATED_ERROR_IDENTIFIERS } from "./mcp/expected-failures" +import { MCP_ANTICIPATED_ERROR_IDENTIFIERS, isAnticipatedMcpFailure } from "./mcp/expected-failures" import { WorkersAiGateway } from "./platform/WorkersAiHttpClient" import { aiPorts, AiBindingsLive, bindAiClients, WorkersAiGatewayLive } from "./worker/bindings" import { buildApp, buildMcpApp, makeFetch } from "./worker/http" @@ -94,6 +94,7 @@ export default MapleAi.make( // `chat/do-runtime.ts` passes the same set to its own tracer. dropSpanNames: ["McpServer/Notifications."], anticipatedErrorIdentifiers: MCP_ANTICIPATED_ERROR_IDENTIFIERS, + isAnticipatedError: isAnticipatedMcpFailure, }), // Read by the bridge's `HttpMiddleware.tracer`; cannot live in the app graph. AlchemyTelemetry.layer(AiObservabilityLive), diff --git a/packages/backend/src/services/integrations/vcs/VcsSourceService.test.ts b/packages/backend/src/services/integrations/vcs/VcsSourceService.test.ts index c10e677785..2e49ae2b31 100644 --- a/packages/backend/src/services/integrations/vcs/VcsSourceService.test.ts +++ b/packages/backend/src/services/integrations/vcs/VcsSourceService.test.ts @@ -1,6 +1,18 @@ import { assert, describe, it } from "@effect/vitest" import { type GitCommitSha, OrgId } from "@maple/domain/http" -import { Deferred, Duration, Effect, Exit, Layer, Option, Schema, Array as Arr, String as Str } from "effect" +import { + Cause, + Deferred, + Duration, + Effect, + Exit, + Layer, + Option, + Predicate, + Schema, + Array as Arr, + String as Str, +} from "effect" import { TestClock } from "effect/testing" import { VcsProviderRegistry } from "./VcsProviderRegistry" import { VcsRepository } from "./VcsRepository" @@ -55,7 +67,8 @@ const makeLayer = (providerCalls: string[], connected: { value: boolean } = { va resolveRef: (_installation, repo, ref) => Effect.sync(() => { providerCalls.push(`resolve:${repo.owner}/${repo.name}:${ref}`) - return ref === "gone" ? Option.none() : Option.some(Str.repeat(40)("d") as GitCommitSha) + if (ref === "gone" || ref === Str.repeat(40)("f")) return Option.none() + return Option.some((/^[0-9a-f]{40}$/.test(ref) ? ref : Str.repeat(40)("d")) as GitCommitSha) }), repositorySizeBytes: (_installation, repo) => Effect.sync(() => { @@ -133,7 +146,7 @@ describe("VcsSourceService", () => { }).pipe(Effect.provide(makeLayer(calls))) }) - it.effect("resolves a checkout at the tracked branch, at a named ref, and straight from a SHA", () => { + it.effect("resolves a checkout at the tracked branch, at a named ref, and at a confirmed SHA", () => { const calls: string[] = [] return Effect.gen(function* () { const source = yield* VcsSourceService @@ -151,8 +164,9 @@ describe("VcsSourceService", () => { assert.isTrue(Exit.isFailure(missing)) assert.deepStrictEqual(calls, [ "resolve:octo/shop:production", - // The second checkout reuses the first one's clone token. "clone:octo/shop", + // The second checkout reuses the first one's clone token. + `resolve:octo/shop:${Str.repeat(40)("e")}`, "resolve:octo/shop:gone", ]) }).pipe(Effect.provide(makeLayer(calls))) @@ -166,11 +180,11 @@ describe("VcsSourceService", () => { const source = yield* VcsSourceService yield* source.resolveCheckout(ORG, "octo/shop", sha) yield* source.resolveCheckout(ORG, "octo/shop", sha) - assert.deepStrictEqual(calls, ["clone:octo/shop"]) + assert.deepStrictEqual(calls, [`resolve:octo/shop:${sha}`, "clone:octo/shop"]) yield* TestClock.adjust(Duration.sum(CLONE_CREDENTIALS_REUSE, Duration.seconds(1))) yield* source.resolveCheckout(ORG, "octo/shop", sha) - assert.deepStrictEqual(calls, ["clone:octo/shop", "clone:octo/shop"]) + assert.deepStrictEqual(calls, [`resolve:octo/shop:${sha}`, "clone:octo/shop", "clone:octo/shop"]) // Disconnected inside the reuse window: the cached token is never handed out. connected.value = false @@ -179,6 +193,25 @@ describe("VcsSourceService", () => { }).pipe(Effect.provide(makeLayer(calls, connected))) }) + it.effect("refuses a SHA the repository does not have, and asks again on the next call", () => { + const calls: string[] = [] + const invented = Str.repeat(40)("f") + return Effect.gen(function* () { + const source = yield* VcsSourceService + const first = yield* Effect.exit(source.resolveCheckout(ORG, "octo/shop", invented)) + const second = yield* Effect.exit(source.resolveCheckout(ORG, "octo/shop", invented)) + const tagOf = (exit: typeof first) => { + const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined + return Predicate.hasProperty(error, "_tag") ? error._tag : undefined + } + assert.deepStrictEqual(Arr.map([first, second], tagOf), [ + "@maple/backend/vcs/VcsSourceRefNotFoundError", + "@maple/backend/vcs/VcsSourceRefNotFoundError", + ]) + assert.deepStrictEqual(calls, [`resolve:octo/shop:${invented}`, `resolve:octo/shop:${invented}`]) + }).pipe(Effect.provide(makeLayer(calls))) + }) + it.effect("does not expose repositories across organizations", () => { const calls: string[] = [] return Effect.gen(function* () { diff --git a/packages/backend/src/services/integrations/vcs/VcsSourceService.ts b/packages/backend/src/services/integrations/vcs/VcsSourceService.ts index abf213f7e3..0ca82e3149 100644 --- a/packages/backend/src/services/integrations/vcs/VcsSourceService.ts +++ b/packages/backend/src/services/integrations/vcs/VcsSourceService.ts @@ -81,6 +81,9 @@ const isCommitSha = Schema.is(GitCommitSha) */ export const CLONE_CREDENTIALS_REUSE = Duration.minutes(30) +/** How long a SHA the provider confirmed is trusted without asking again. */ +export const KNOWN_COMMIT_REUSE = Duration.hours(1) + export interface ConnectedSourceRepository { readonly provider: VcsRepo["provider"] readonly fullName: string @@ -228,6 +231,29 @@ export class VcsSourceService extends Context.Service + asUpstream(providers.resolve(key.provider)).pipe( + Effect.flatMap((provider) => + asUpstream(provider.resolveRef(key.installation, key.repository, key.sha)), + ), + ), + { + capacity: 512, + timeToLive: (exit) => + Exit.isSuccess(exit) && Option.isSome(exit.value) + ? KNOWN_COMMIT_REUSE + : Duration.zero, + }, + ) + const resolveRepository = Effect.fn("VcsSourceService.resolveRepository")(function* ( orgId: OrgId, fullName: string, @@ -457,9 +483,15 @@ export class VcsSourceService extends Context.Service) => + PG.sql(PG.array(PG.float4))`${`{${Arr.join(Arr.map(vector, String), ",")}}`}::float4[]` + /** * What the review posts: a check run with every finding as an annotation, a summary comment of * its own that replaces its "reviewing" notice, and inline comments for new @@ -2912,7 +2919,7 @@ export class PrReviewService extends Context.Service | undefined + /** Expected failures a tag alone cannot single out, judged on the failure value. */ + // BOUNDARY: the tracer hands over a cause's raw failure values. + readonly isAnticipatedError?: ((error: unknown) => boolean) | undefined /** Span-name prefixes never exported. */ readonly dropSpanNames?: ReadonlyArray | undefined } @@ -91,6 +94,7 @@ export const workerTelemetryConfig = (options: WorkerTelemetryOptions): Telemetr ...ANTICIPATED_ERROR_IDENTIFIERS, ...(options.anticipatedErrorIdentifiers ?? []), ], + isAnticipatedError: options.isAnticipatedError, }) export const WorkerTelemetry = (options: WorkerTelemetryOptions): Layer.Layer => diff --git a/packages/query-engine/src/execution/errors.test.ts b/packages/query-engine/src/execution/errors.test.ts index af3abab410..3b2b6c8970 100644 --- a/packages/query-engine/src/execution/errors.test.ts +++ b/packages/query-engine/src/execution/errors.test.ts @@ -245,6 +245,135 @@ describe("mapWarehouseError", () => { }) }) + describe("caller SQL the database refused", () => { + // Shapes from production run_sql spans: each went out as a server error issue. + it.each([ + [ + "NOT_IMPLEMENTED", + "[Error] Correlated subqueries in aggregate function argument are not supported.", + ], + ["CANNOT_COMPILE_REGEXP", "[Error] OptimizedRegularExpression: cannot compile re2: {0,2500}"], + ["SIZES_OF_ARRAYS_DONT_MATCH", "[Error] Arrays passed to arrayMap must have equal size."], + ["UNSUPPORTED_METHOD", "[Error] Qualified matcher turns.* found a non-compound expression"], + ["CANNOT_PARSE_NUMBER", "[Error] Cannot parse Float64 from String"], + ])("reads a %s as invalid SQL", (type, message) => { + expect(classify("rawSqlQuery", { message, type, status: 400 }, "caller")).toBeInstanceOf( + WarehouseInvalidSqlError, + ) + }) + + it("reads an unknown column as invalid SQL even when the echoed SQL looks like a network failure", () => { + const message = + "[Error] Unknown expression or function identifier `DeploymentEnv` in scope SELECT count() FROM traces WHERE StatusMessage LIKE '%fetch failed%'" + expect(classify("rawSqlQuery", { message, type: "UNKNOWN_IDENTIFIER" }, "caller")).toBeInstanceOf( + WarehouseInvalidSqlError, + ) + }) + + it("reads a 400 with a bare exception code as invalid SQL", () => { + const mapped = classify( + "rawSqlQuery", + { + message: + "HTTP 400: [Error] Some columns need to be aggregated by using the -Merge suffix.", + code: "43", + status: 400, + }, + "caller", + ) + expect(mapped).toBeInstanceOf(WarehouseInvalidSqlError) + }) + + it("reads a table Maple never had as invalid SQL, and a missing Maple table as configuration", () => { + expect( + classify( + "rawSqlQuery", + { message: "HTTP 403: Resource 'traces_hourly' not found", status: 403 }, + "caller", + ), + ).toBeInstanceOf(WarehouseInvalidSqlError) + expect( + classify( + "rawSqlQuery", + { message: "HTTP 403: Resource 'profiles' not found", status: 403 }, + "caller", + ), + ).toBeInstanceOf(WarehouseConfigError) + }) + + it("keeps the same refusal of Maple's own SQL a query error", () => { + expect( + classify( + "p", + { + message: "[Error] Arrays passed to arrayMap must have equal size.", + type: "SIZES_OF_ARRAYS_DONT_MATCH", + }, + "maple", + ), + ).toBeInstanceOf(WarehouseQueryError) + }) + + it("keeps an engine fault on caller SQL a server-side query error", () => { + expect( + classify( + "rawSqlQuery", + { message: "[Error] Logical error: bad cast", type: "LOGICAL_ERROR" }, + "caller", + ), + ).toBeInstanceOf(WarehouseQueryError) + }) + + it.each([ + ["MEMORY_LIMIT_EXCEEDED", undefined], + ["TOO_MANY_PARTS", undefined], + ["NOT_ENOUGH_SPACE", undefined], + ["CANNOT_ALLOCATE_MEMORY", undefined], + ["UNKNOWN_EXCEPTION", undefined], + [undefined, "241"], + [undefined, "159"], + ])("never reads a server-side failure on caller SQL (%s %s) as invalid SQL", (type, code) => { + const mapped = classify( + "rawSqlQuery", + { message: "HTTP 400: [Error] the cluster gave up. status: 400", type, code, status: 400 }, + "caller", + ) + expect(mapped).not.toBeInstanceOf(WarehouseInvalidSqlError) + }) + + it("keeps a timeout whose message echoes a missing table upstream", () => { + expect( + classify( + "rawSqlQuery", + { + message: "timed out running: SELECT 1 -- Resource 'spans' not found", + type: "SOCKET_TIMEOUT", + }, + "caller", + ), + ).toBeInstanceOf(WarehouseUpstreamError) + }) + + it("keeps an engine fault whose message echoes a missing table a query error", () => { + expect( + classify( + "p", + { + message: "[Error] Logical error near: -- Resource 'spans' not found", + type: "LOGICAL_ERROR", + }, + "maple", + ), + ).toBeInstanceOf(WarehouseQueryError) + }) + + it("keeps transient failures upstream for caller SQL", () => { + expect( + classify("rawSqlQuery", { message: "blip", type: "SOCKET_TIMEOUT" }, "caller"), + ).toBeInstanceOf(WarehouseUpstreamError) + }) + }) + describe("auth", () => { it("classifies the auth ClickHouse type", () => { expect(classify("p", { message: "nope", type: "AUTHENTICATION_FAILED" })).toBeInstanceOf( diff --git a/packages/query-engine/src/execution/errors.ts b/packages/query-engine/src/execution/errors.ts index d0f874844d..4047bf74c0 100644 --- a/packages/query-engine/src/execution/errors.ts +++ b/packages/query-engine/src/execution/errors.ts @@ -16,7 +16,9 @@ import { } from "@maple/domain/http" import type { WarehouseDriverError, WarehouseDriverFailureReason } from "./driver-error" import { detectQuotaSetting } from "../profiles" -import { String as Str, Option } from "effect" +import { Array as Arr, Option, Record as Rec, String as Str } from "effect" +import { isDatasourceDefinition } from "@maple-dev/effect-orm/tinybird" +import * as Datasources from "@maple/domain/tinybird" import { indexOfFrom } from "@maple/strings" /** @@ -160,21 +162,114 @@ type ClassificationRule = { readonly types?: ReadonlySet readonly pattern?: RegExp readonly reasons?: ReadonlySet + /** Any other condition, given what the database answered. */ + readonly test?: (answer: DatabaseAnswer) => boolean /** Restricts the rule to SQL with this authorship. Unset means either. */ readonly authoredBy?: SqlAuthorship /** Construct the tagged error for this rule. `upstreamStatus` is only used by the rules that carry it. */ readonly make: (base: ClassifiedBase, upstreamStatus: number | undefined) => WarehouseClassifiedError } +type DatabaseAnswer = { + readonly rawMessage: string + readonly code: string | undefined + readonly ruleType: string | undefined + readonly upstreamStatus: number | undefined +} + +const MISSING_RESOURCE = /Resource '([^']*)' not found/i + +/** Every table Maple defines, the ones raw SQL may not read included. */ +const MAPLE_DATASOURCE_NAMES: ReadonlySet = new Set( + Arr.map(Arr.filter(Rec.values(Datasources), isDatasourceDefinition), (ds) => ds._name), +) + +/** + * Exception types that are only ever about the statement itself (its syntax, names, types or + * arguments). An allow-list: an unlisted type (a capacity limit, an engine fault, a new code) + * stays a server-side query error, never the caller's mistake. + */ +const STATEMENT_TYPES: ReadonlyMap = new Map([ + ["CANNOT_PARSE_TEXT", "6"], + ["THERE_IS_NO_COLUMN", "8"], + ["DUPLICATE_COLUMN", "15"], + ["NO_SUCH_COLUMN_IN_TABLE", "16"], + ["BAD_ARGUMENTS", "36"], + ["CANNOT_PARSE_DATE", "38"], + ["CANNOT_PARSE_DATETIME", "41"], + ["NUMBER_OF_ARGUMENTS_DOESNT_MATCH", "42"], + ["ILLEGAL_TYPE_OF_ARGUMENT", "43"], + ["ILLEGAL_COLUMN", "44"], + ["UNKNOWN_FUNCTION", "46"], + ["UNKNOWN_IDENTIFIER", "47"], + ["NOT_IMPLEMENTED", "48"], + ["UNKNOWN_TYPE", "50"], + ["TYPE_MISMATCH", "53"], + ["UNKNOWN_TABLE", "60"], + ["SYNTAX_ERROR", "62"], + ["UNKNOWN_AGGREGATE_FUNCTION", "63"], + ["ARGUMENT_OUT_OF_BOUND", "69"], + ["CANNOT_CONVERT_TYPE", "70"], + ["CANNOT_PARSE_NUMBER", "72"], + ["INCORRECT_QUERY", "80"], + ["UNKNOWN_DATABASE", "81"], + ["ILLEGAL_DIVISION", "153"], + ["ILLEGAL_AGGREGATION", "184"], + ["SIZES_OF_ARRAYS_DONT_MATCH", "190"], + ["AMBIGUOUS_IDENTIFIER", "207"], + ["NOT_AN_AGGREGATE", "215"], + ["AMBIGUOUS_COLUMN_NAME", "352"], + ["NO_COMMON_TYPE", "386"], + ["CANNOT_COMPILE_REGEXP", "427"], + ["INVALID_JOIN_ON_EXPRESSION", "403"], + ["CANNOT_READ_ARRAY_FROM_TEXT", "130"], +]) + +const STATEMENT_CODES: ReadonlySet = new Set(STATEMENT_TYPES.values()) + +/** UNSUPPORTED_METHOD also covers server-side limits; only this analyzer complaint is the statement's. */ +const QUALIFIED_MATCHER = /Qualified matcher .* found a non-compound expression/i + +/** + * The database parsed a caller's statement and refused it: a statement-level exception type, or + * a bare statement-level code (`Code: 43`). The status is never read: it can be scraped from + * message text that echoes the caller's SQL. + */ +const refusedStatement = ({ code, ruleType, rawMessage }: DatabaseAnswer): boolean => + ruleType === "UNSUPPORTED_METHOD" + ? QUALIFIED_MATCHER.test(rawMessage) + : ruleType !== undefined + ? STATEMENT_TYPES.has(ruleType) + : code !== undefined && STATEMENT_CODES.has(code) + // Ordered rules — first match wins. A raw error can satisfy several patterns // (e.g. a 503 carrying an HTML body, or schema-drift text inside a transient // failure), so the order encodes precedence: -// missing datasource > auth > upstream > config > client > schema_drift > (default) query. +// missing datasource > auth > refused caller SQL > upstream > config > client > schema_drift > (default) query. const CLASSIFICATION_RULES: ReadonlyArray = [ + { + // A caller naming a table Maple never had (`FROM spans`, a CTE Tinybird cannot resolve) gets + // the same 403 as an undeployed datasource; only a Maple table missing is configuration. + authoredBy: "caller", + // Only a gateway answer with no ClickHouse identity, or a statement refusal: the message can + // echo the caller's SQL, so a timeout quoting `Resource 'x' not found` must not match. + test: (answer) => { + const name = MISSING_RESOURCE.exec(answer.rawMessage)?.[1] + const hasIdentity = answer.code !== undefined || answer.ruleType !== undefined + return ( + name !== undefined && + !MAPLE_DATASOURCE_NAMES.has(name) && + (!hasIdentity || refusedStatement(answer)) + ) + }, + make: (base) => new WarehouseInvalidSqlError(base), + }, { // Tinybird answers a missing datasource with a 403, so the auth rule's status match read a - // table still being deployed as rejected credentials. The text names the failure; it wins. - pattern: /Resource '[^']*' not found/i, + // table still being deployed as rejected credentials. The text names the failure; it wins, + // unless a ClickHouse identity says the text is only echoed SQL. + test: ({ rawMessage, code, ruleType }) => + MISSING_RESOURCE.test(rawMessage) && code === undefined && ruleType === undefined, make: (base) => new WarehouseConfigError(base), }, { @@ -184,6 +279,13 @@ const CLASSIFICATION_RULES: ReadonlyArray = [ /authentication failed|access denied|not enough privileges|password is incorrect|invalid authentication token/i, make: (base, upstreamStatus) => new WarehouseAuthError({ ...base, upstreamStatus }), }, + { + // Before the upstream rule: its patterns match anywhere in the message, which echoes the + // caller's SQL, so `LIKE '%fetch failed%'` read an unknown column as a network failure. + authoredBy: "caller", + test: refusedStatement, + make: (base) => new WarehouseInvalidSqlError(base), + }, { status: (s) => s === 408 || s === 429 || (s >= 500 && s < 600), reasons: new Set(["transport"]), @@ -220,11 +322,10 @@ const CLASSIFICATION_RULES: ReadonlyArray = [ { status: (s) => s === 404, types: new Set(["UNKNOWN_DATABASE", "UNKNOWN_TABLE", "TABLE_IS_DROPPED", "UNKNOWN_SETTING"]), - // "Resource '' not found" is the Tinybird gateway's phrasing of - // UNKNOWN_TABLE; without it a missing datasource fell through to the - // generic query error and the per-org missing-table fallbacks never fired. + // Tinybird's "Resource '' not found" is claimed by the missing-resource rule above, only + // without a ClickHouse identity, so a typed error that merely echoes it is not read as config. pattern: - /Invalid URL|unknown database|unknown table|table .* does not exist|database .* does not exist|Resource '[^']*' not found/i, + /Invalid URL|unknown database|unknown table|table .* does not exist|database .* does not exist/i, make: (base) => new WarehouseConfigError(base), }, { @@ -415,7 +516,8 @@ export const mapWarehouseError = ( upstreamStatus !== undefined && rule.status(upstreamStatus)) || (rule.types !== undefined && ruleType !== undefined && rule.types.has(ruleType)) || - (rule.pattern !== undefined && rule.pattern.test(rawMessage)) + (rule.pattern !== undefined && rule.pattern.test(rawMessage)) || + (rule.test !== undefined && rule.test({ rawMessage, code, ruleType, upstreamStatus })) if (matches) return rule.make(base, upstreamStatus) } return new WarehouseQueryError(base)