Skip to content
3 changes: 2 additions & 1 deletion apps/ai/src/chat/do-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { type ChatSessionNamespace, chatSessionsLayerIfBound } from "@maple/back
import { envPorts } from "@maple/backend/platform/env-ports"
import { runtimeTelemetry } from "@maple/infra/worker-telemetry"
import { Effect, Layer, Schema } from "effect"
import { MCP_ANTICIPATED_ERROR_IDENTIFIERS } from "../mcp/expected-failures"
import { MCP_ANTICIPATED_ERROR_IDENTIFIERS, isAnticipatedMcpFailure } from "../mcp/expected-failures"
import type { AiSessionPortsLayer } from "../worker/bindings"
import { provideSessionPorts } from "./session-ports"

Expand Down Expand Up @@ -46,6 +46,7 @@ const ENGINE_BOOKKEEPING_SPANS = [
export const chatTelemetry = runtimeTelemetry({
serviceName: "maple-chat",
anticipatedErrorIdentifiers: MCP_ANTICIPATED_ERROR_IDENTIFIERS,
isAnticipatedError: isAnticipatedMcpFailure,
dropSpanNames: ENGINE_BOOKKEEPING_SPANS,
})

Expand Down
84 changes: 84 additions & 0 deletions apps/ai/src/mcp/expected-failures.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import { describe, expect, it } from "vitest"
import { Duration, String as Str } from "effect"
import { IntegrationsPersistenceError, IntegrationsUpstreamError } from "@maple/domain/http"
import { SandboxRunCheckoutFailed, SandboxRunCheckoutPending } from "@maple/domain/sandbox"
import {
SandboxExitError,
SandboxImplementation,
SandboxOutputLimitError,
SandboxSpawnError,
SandboxTimeoutError,
SandboxUnsupportedRequestError,
type SandboxError,
} from "@maple/backend/services/sandbox/contract"
import { VcsSourceRefNotFoundError } from "@maple/backend/services/integrations/vcs/VcsSourceService"
import { isAnticipatedMcpFailure } from "./expected-failures"
import { fromSandboxError } from "./lib/source-errors"

const implementation = new SandboxImplementation({ isolation: "isolated", identity: "test" })
const spawn = (cause: unknown): SandboxError =>
new SandboxSpawnError({ implementation, command: "git", message: "never started", cause })

const SANDBOX_FAILURES: ReadonlyArray<readonly [string, SandboxError]> = [
[
"an invented SHA",
spawn(
new VcsSourceRefNotFoundError({ repository: "o/r", ref: Str.repeat(40)("a"), message: "no ref" }),
),
],
["a clone still running", spawn(new SandboxRunCheckoutPending({ message: "cloning" }))],
[
"a clone that failed",
spawn(new SandboxRunCheckoutFailed({ message: "Cloning o/r failed (exit 128)" })),
],
[
"a GitHub rate limit on the checkout lookup",
spawn(
new IntegrationsUpstreamError({ message: "GitHub rate limited (retry after 60s)", status: 403 }),
),
],
[
"a GitHub outage on the checkout lookup",
spawn(new IntegrationsUpstreamError({ message: "Get commit failed: 502", status: 502 })),
],
["a lost database connection", spawn(new IntegrationsPersistenceError({ message: "Connection closed" }))],
["a timeout", new SandboxTimeoutError({ implementation, maxWallTime: Duration.seconds(30) })],
[
"too much output",
new SandboxOutputLimitError({ implementation, stream: "stdout", limit: 1024, observed: 2048 }),
],
[
"a bad mount",
new SandboxUnsupportedRequestError({
implementation,
feature: "mounts",
message: "bad mount source",
}),
],
[
"no sandbox runtime",
new SandboxUnsupportedRequestError({ implementation, feature: "runtime", message: "no runtime" }),
],
["a command that exited", new SandboxExitError({ implementation, exitCode: 2, message: "exit 2" })],
]

describe("isAnticipatedMcpFailure", () => {
// The span and the model must agree: what the model is told to fix or retry is not a server error.
// A missing capability (no sandbox runtime) is told to the model too, but stays an Error span.
it.each(SANDBOX_FAILURES)("agrees with the model-facing mapping for %s", (_, error) => {
const shownToModel = fromSandboxError("sandbox_exec")(error)._tag
expect(isAnticipatedMcpFailure(error)).toBe(
shownToModel === "@maple/mcp/errors/McpInvalidInputError" ||
shownToModel === "@maple/mcp/errors/McpNotReadyError",
)
})

it("treats a call to a tool that does not exist as the client's mistake", () => {
const unknownTool = { _tag: "ProtocolError", code: -32602, message: "Tool 'tools/list' not found" }
const internal = { _tag: "ProtocolError", code: -32603, message: "Internal error" }
expect([isAnticipatedMcpFailure(unknownTool), isAnticipatedMcpFailure(internal)]).toEqual([
true,
false,
])
})
})
58 changes: 57 additions & 1 deletion apps/ai/src/mcp/expected-failures.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Effect, Record as Rec, String as Str } from "effect"
import { Effect, Predicate, Record as Rec, String as Str } from "effect"

/**
* MCP failures that are expected 4xx client outcomes, not bugs: a bad or absent
Expand Down Expand Up @@ -37,6 +37,62 @@ const expectedStatusByTag = new Map<string, number>(Rec.toEntries(MCP_EXPECTED_F
/** Spread into `anticipatedErrorIdentifiers` next to `ANTICIPATED_ERROR_IDENTIFIERS`. */
export const MCP_ANTICIPATED_ERROR_IDENTIFIERS: ReadonlyArray<string> = [...expectedStatusByTag.keys()]

/** JSON-RPC codes for a request the client got wrong: an unknown method, or params (a tool name) that name nothing. */
const CLIENT_PROTOCOL_CODES: ReadonlySet<unknown> = new Set([-32601, -32602])

/** Why a repository command never started, when the reason is the call's ref, repository or path. */
const CALLER_SPAWN_CAUSES: ReadonlySet<string | undefined> = new Set([
"SandboxRunCheckoutPending",
"@maple/backend/vcs/VcsSourceRefNotFoundError",
"@maple/backend/vcs/VcsSourceRepositoryNotFoundError",
"@maple/backend/vcs/VcsSourceFileNotFoundError",
])

// BOUNDARY: a failure value from the tracer, or the cause a sandbox error carries.
/** The GitHub client's wording for a rate limit, which carries its wait in seconds. */
const GITHUB_RATE_LIMITED = /^GitHub rate limited \(retry after (\d+)s\)/

/** The wait a GitHub rate limit asks for, when `message` is one. */
export const githubRateLimitWait = (message: string): number | undefined => {
const seconds = GITHUB_RATE_LIMITED.exec(message)?.[1]
return seconds === undefined ? undefined : Number(seconds)
}

const isRateLimitedLookup = (cause: unknown): boolean =>
tagOf(cause) === "@maple/http/errors/IntegrationsUpstreamError" &&
Predicate.hasProperty(cause, "message") &&
typeof cause.message === "string" &&
githubRateLimitWait(cause.message) !== undefined

const tagOf = (value: unknown): string | undefined =>
Predicate.hasProperty(value, "_tag") && typeof value._tag === "string" ? value._tag : undefined

/**
* Failures a tag cannot single out, judged on the value: the SDK's MCP protocol error for a call to
* a tool that does not exist, and repository sandbox failures `fromSandboxError` hands the model as
* a fixable call or one to retry later (a missing capability stays an Error). Both export `Ok`.
*
* BOUNDARY: the tracer hands over a cause's raw failure values.
*/
export const isAnticipatedMcpFailure = (error: unknown): boolean => {
switch (tagOf(error)) {
case "ProtocolError":
return Predicate.hasProperty(error, "code") && CLIENT_PROTOCOL_CODES.has(error.code)
case "@maple/backend/sandbox/SandboxTimeoutError":
case "@maple/backend/sandbox/SandboxOutputLimitError":
return true
case "@maple/backend/sandbox/SandboxUnsupportedRequestError":
return Predicate.hasProperty(error, "feature") && error.feature !== "runtime"
case "@maple/backend/sandbox/SandboxSpawnError":
return (
Predicate.hasProperty(error, "cause") &&
(CALLER_SPAWN_CAUSES.has(tagOf(error.cause)) || isRateLimitedLookup(error.cause))
)
default:
return false
}
}

/**
* Longest failure message a span or a transcript keeps. A ClickHouse syntax error arrives with the
* whole offending query inlined, and on a retry loop it would land once per attempt.
Expand Down
35 changes: 35 additions & 0 deletions apps/ai/src/mcp/lib/source-errors.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { describe, expect, it } from "vitest"
import { IntegrationsUpstreamError } from "@maple/domain/http"
import { fromVcsLookupError } from "./source-errors"

const mapped = (error: IntegrationsUpstreamError) => fromVcsLookupError("search_source_code")(error)

describe("fromVcsLookupError", () => {
it("asks the model to come back after a GitHub rate limit", () => {
const error = mapped(
new IntegrationsUpstreamError({ message: "GitHub rate limited (retry after 60s)", status: 403 }),
)
expect(error).toMatchObject({ _tag: "@maple/mcp/errors/McpNotReadyError", retryAfterSeconds: 60 })
})

it("hands a search query GitHub could not parse back to the model to fix", () => {
const error = mapped(
new IntegrationsUpstreamError({
message:
'Search code failed: 422 {"message":"ERROR_TYPE_QUERY_PARSING_FATAL unable to parse query!"}',
status: 422,
}),
)
expect(error._tag).toBe("@maple/mcp/errors/McpInvalidInputError")
})

it("keeps a GitHub outage a query failure", () => {
const error = mapped(
new IntegrationsUpstreamError({
message: "Get pull request failed: 503 Service Unavailable",
status: 503,
}),
)
expect(error._tag).toBe("@maple/mcp/errors/McpQueryError")
})
})
14 changes: 10 additions & 4 deletions apps/ai/src/mcp/lib/source-errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
VcsSourceRepositoryNotFoundError,
} from "@maple/backend/services/integrations/vcs/VcsSourceService"
import { McpInvalidInputError, McpNotReadyError, McpQueryError, McpUnavailableError } from "../tools/types"
import { githubRateLimitWait } from "../expected-failures"

export type VcsLookupError =
| IntegrationsNotConnectedError
Expand Down Expand Up @@ -72,10 +73,15 @@ export const fromVcsLookupError =
message: "No source repository is connected to this organization.",
capability: "source_repository",
})
case "@maple/http/errors/IntegrationsUpstreamError":
return isGithubAppMissing(error)
? GITHUB_APP_UNAVAILABLE
: new McpQueryError({ message: error.message, pipeName: operation, cause: error })
case "@maple/http/errors/IntegrationsUpstreamError": {
if (isGithubAppMissing(error)) return GITHUB_APP_UNAVAILABLE
const retryAfter = githubRateLimitWait(error.message)
if (retryAfter !== undefined)
return new McpNotReadyError({ message: error.message, retryAfterSeconds: retryAfter })
// GitHub could not parse what the call sent (a code search query, most often).
if (error.status === 422) return new McpInvalidInputError({ message: error.message })
return new McpQueryError({ message: error.message, pipeName: operation, cause: error })
}
case "@maple/http/errors/IntegrationsPersistenceError":
return new McpQueryError({ message: error.message, pipeName: operation, cause: error })
}
Expand Down
18 changes: 18 additions & 0 deletions apps/ai/src/platform/Llm.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,13 @@ import { FetchHttpClient } from "effect/http"
import { assert, describe, it } from "@effect/vitest"
import { expect } from "vitest"
import { PR_REVIEW_MODELS } from "@maple/domain/http"
import { FindingEmbedder } from "@maple/backend/services/pr-review/FindingEmbedder"
import {
DEFAULT_DECISION_MODEL,
layerDecisionModel,
DEFAULT_REVIEW_MODEL,
DEFAULT_EMBEDDING_MODEL,
layerFindingEmbedder,
layerLlm,
loadLlmSettings,
resolveDecisionModel,
Expand Down Expand Up @@ -518,6 +521,21 @@ describe("EU in-region routing", () => {
})
})

describe("layerFindingEmbedder", () => {
const embedderFor = (settings: LlmSettings) =>
Effect.runSync(
Effect.gen(function* () {
const embedder = yield* FindingEmbedder
return Option.map(embedder, (found) => found.model)
}).pipe(Effect.provide(layerFindingEmbedder(settings))),
)

it("embeds on OpenRouter in the US and not in the EU, where no provider serves the model", () => {
expect(embedderFor(openRouterEnv)).toEqual(Option.some(DEFAULT_EMBEDDING_MODEL))
expect(embedderFor({ ...openRouterEnv, MAPLE_REGION: "eu" })).toEqual(Option.none())
})
})

describe("resolveTriageModel — context limits", () => {
it("attaches the configured model's window, which upstream leaves unstated", () => {
// Providers do not report their context window, so without this table nothing could tell
Expand Down
5 changes: 3 additions & 2 deletions apps/ai/src/platform/Llm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -831,11 +831,12 @@ export const resolveDecisionModel = (settings: LlmSettings): string | undefined
* The embedder the PR review's feedback filter compares findings with, on OpenRouter whichever
* provider the agents run on: Workers AI's binding shim only answers chat. Its `OpenAiClient` is
* private to this layer, so it never replaces the Workers AI one `layerLlm` provides. Without an
* OpenRouter key there is no embedder, and the filter is off.
* OpenRouter key there is no embedder, and the filter is off. Nor in the EU: OpenRouter's EU
* endpoint serves no provider for the embedding model ("No endpoints found supporting your data region").
*/
export const layerFindingEmbedder = (settings: LlmSettings): Layer.Layer<FindingEmbedder> => {
const apiKey = readSecret(settings, "OPENROUTER_API_KEY")
if (apiKey === undefined) return FindingEmbedder.none
if (apiKey === undefined || openRouterRegion(settings) === "eu") return FindingEmbedder.none
const model = DEFAULT_EMBEDDING_MODEL
return Layer.effect(FindingEmbedder)(
Effect.map(OpenAiEmbeddingModel.make({ model }), (embeddings) =>
Expand Down
3 changes: 2 additions & 1 deletion apps/ai/src/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { ChatSessionObject } from "@maple/backend/platform/chat-sessions"
import { ChatSessionLive } from "./chat/ChatSession"
import { aiConfiguredEnv } from "./resources/env"
import SecurityAuditWorkflow from "./security-audit/SecurityAuditWorkflow"
import { MCP_ANTICIPATED_ERROR_IDENTIFIERS } from "./mcp/expected-failures"
import { MCP_ANTICIPATED_ERROR_IDENTIFIERS, isAnticipatedMcpFailure } from "./mcp/expected-failures"
import { WorkersAiGateway } from "./platform/WorkersAiHttpClient"
import { aiPorts, AiBindingsLive, bindAiClients, WorkersAiGatewayLive } from "./worker/bindings"
import { buildApp, buildMcpApp, makeFetch } from "./worker/http"
Expand Down Expand Up @@ -94,6 +94,7 @@ export default MapleAi.make(
// `chat/do-runtime.ts` passes the same set to its own tracer.
dropSpanNames: ["McpServer/Notifications."],
anticipatedErrorIdentifiers: MCP_ANTICIPATED_ERROR_IDENTIFIERS,
isAnticipatedError: isAnticipatedMcpFailure,
}),
// Read by the bridge's `HttpMiddleware.tracer`; cannot live in the app graph.
AlchemyTelemetry.layer(AiObservabilityLive),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,18 @@
import { assert, describe, it } from "@effect/vitest"
import { type GitCommitSha, OrgId } from "@maple/domain/http"
import { Deferred, Duration, Effect, Exit, Layer, Option, Schema, Array as Arr, String as Str } from "effect"
import {
Cause,
Deferred,
Duration,
Effect,
Exit,
Layer,
Option,
Predicate,
Schema,
Array as Arr,
String as Str,
} from "effect"
import { TestClock } from "effect/testing"
import { VcsProviderRegistry } from "./VcsProviderRegistry"
import { VcsRepository } from "./VcsRepository"
Expand Down Expand Up @@ -55,7 +67,8 @@ const makeLayer = (providerCalls: string[], connected: { value: boolean } = { va
resolveRef: (_installation, repo, ref) =>
Effect.sync(() => {
providerCalls.push(`resolve:${repo.owner}/${repo.name}:${ref}`)
return ref === "gone" ? Option.none() : Option.some(Str.repeat(40)("d") as GitCommitSha)
if (ref === "gone" || ref === Str.repeat(40)("f")) return Option.none()
return Option.some((/^[0-9a-f]{40}$/.test(ref) ? ref : Str.repeat(40)("d")) as GitCommitSha)
}),
repositorySizeBytes: (_installation, repo) =>
Effect.sync(() => {
Expand Down Expand Up @@ -133,7 +146,7 @@ describe("VcsSourceService", () => {
}).pipe(Effect.provide(makeLayer(calls)))
})

it.effect("resolves a checkout at the tracked branch, at a named ref, and straight from a SHA", () => {
it.effect("resolves a checkout at the tracked branch, at a named ref, and at a confirmed SHA", () => {
const calls: string[] = []
return Effect.gen(function* () {
const source = yield* VcsSourceService
Expand All @@ -151,8 +164,9 @@ describe("VcsSourceService", () => {
assert.isTrue(Exit.isFailure(missing))
assert.deepStrictEqual(calls, [
"resolve:octo/shop:production",
// The second checkout reuses the first one's clone token.
"clone:octo/shop",
// The second checkout reuses the first one's clone token.
`resolve:octo/shop:${Str.repeat(40)("e")}`,
"resolve:octo/shop:gone",
])
}).pipe(Effect.provide(makeLayer(calls)))
Expand All @@ -166,11 +180,11 @@ describe("VcsSourceService", () => {
const source = yield* VcsSourceService
yield* source.resolveCheckout(ORG, "octo/shop", sha)
yield* source.resolveCheckout(ORG, "octo/shop", sha)
assert.deepStrictEqual(calls, ["clone:octo/shop"])
assert.deepStrictEqual(calls, [`resolve:octo/shop:${sha}`, "clone:octo/shop"])

yield* TestClock.adjust(Duration.sum(CLONE_CREDENTIALS_REUSE, Duration.seconds(1)))
yield* source.resolveCheckout(ORG, "octo/shop", sha)
assert.deepStrictEqual(calls, ["clone:octo/shop", "clone:octo/shop"])
assert.deepStrictEqual(calls, [`resolve:octo/shop:${sha}`, "clone:octo/shop", "clone:octo/shop"])

// Disconnected inside the reuse window: the cached token is never handed out.
connected.value = false
Expand All @@ -179,6 +193,25 @@ describe("VcsSourceService", () => {
}).pipe(Effect.provide(makeLayer(calls, connected)))
})

it.effect("refuses a SHA the repository does not have, and asks again on the next call", () => {
const calls: string[] = []
const invented = Str.repeat(40)("f")
return Effect.gen(function* () {
const source = yield* VcsSourceService
const first = yield* Effect.exit(source.resolveCheckout(ORG, "octo/shop", invented))
const second = yield* Effect.exit(source.resolveCheckout(ORG, "octo/shop", invented))
const tagOf = (exit: typeof first) => {
const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined
return Predicate.hasProperty(error, "_tag") ? error._tag : undefined
}
assert.deepStrictEqual(Arr.map([first, second], tagOf), [
"@maple/backend/vcs/VcsSourceRefNotFoundError",
"@maple/backend/vcs/VcsSourceRefNotFoundError",
])
assert.deepStrictEqual(calls, [`resolve:octo/shop:${invented}`, `resolve:octo/shop:${invented}`])
}).pipe(Effect.provide(makeLayer(calls)))
})

it.effect("does not expose repositories across organizations", () => {
const calls: string[] = []
return Effect.gen(function* () {
Expand Down
Loading
Loading