From 2e99b3c3b0a3541f2a5de370c3751090499664db Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 16 Nov 2026 17:24:41 +0100 Subject: [PATCH 1/2] fix(ai): investigations finish inside their clock and close out inside the context limit - A model call past its 90s attempt cap keeps running while the low-effort retry starts, and whichever answers first wins. Cutting it threw away 90s, and the retry took 83s more at the median. - Investigation passes get 15 minutes. At 10, 17% of passes ran out of clock after 9-17 model calls at about 45 tokens a second. - A close-out runs on its own 5-minute clock, so pass plus close-out stays under the turn watchdog. - The close-out's evidence drops to 120k characters. At 200k it opened at 123k of 128k tokens and the round after submit_diagnosis no longer fit. --- apps/ai/src/chat/autonomous-phases.test.ts | 17 ++++++- apps/ai/src/chat/autonomous-phases.ts | 14 ++++++ apps/ai/src/chat/budgets.test.ts | 8 +++- apps/ai/src/chat/budgets.ts | 14 +++++- apps/ai/src/chat/close-out.test.ts | 14 ++++++ apps/ai/src/chat/close-out.ts | 8 +++- apps/ai/src/platform/genai-spans.ts | 50 +++++++++++++------- apps/ai/src/platform/model-call-span.test.ts | 30 ++++++++++++ 8 files changed, 131 insertions(+), 24 deletions(-) diff --git a/apps/ai/src/chat/autonomous-phases.test.ts b/apps/ai/src/chat/autonomous-phases.test.ts index 5bf7d34d74..2c1e429d98 100644 --- a/apps/ai/src/chat/autonomous-phases.test.ts +++ b/apps/ai/src/chat/autonomous-phases.test.ts @@ -1,7 +1,8 @@ -import { Effect, Array as Arr } from "effect" +import { Duration, Effect, Array as Arr } from "effect" import { assert, describe, it } from "vitest" import { AGENTS, type RowMode } from "./agents" import { makeAutonomousOutcome } from "./autonomous-outcome" +import { CLOSE_OUT_MAX_DURATION } from "./budgets" import { type PhaseTurn, runAutonomousPhases } from "./autonomous-phases" /** Runs the phases over a scripted run: each phase files when `files` says so, and ends on `stop`. */ @@ -57,6 +58,20 @@ describe("runAutonomousPhases", () => { assert.isFalse(outcome.submitted()) }) + it("runs an investigation's close-out on its own short clock, not the pass's", async () => { + const { turns } = await phasesOf("investigate", (turn) => turn.closeOut === true) + const clock = (turn: PhaseTurn | undefined) => + turn?.agent === undefined ? undefined : Duration.toMillis(turn.agent.budget.maxDuration) + assert.strictEqual( + clock(turns[1]), + Duration.toMillis(Duration.fromInputUnsafe(CLOSE_OUT_MAX_DURATION)), + ) + assert.isBelow( + clock(turns[1]) ?? Infinity, + Duration.toMillis(Duration.fromInputUnsafe(AGENTS.investigate.budget.maxDuration)), + ) + }) + it("closes a reply out without a continuation, which only a review has", async () => { const { turns } = await phasesOf("pr-reply", () => false) assert.deepEqual( diff --git a/apps/ai/src/chat/autonomous-phases.ts b/apps/ai/src/chat/autonomous-phases.ts index 287efb2565..d756f6e054 100644 --- a/apps/ai/src/chat/autonomous-phases.ts +++ b/apps/ai/src/chat/autonomous-phases.ts @@ -11,6 +11,7 @@ import { summarizeCause } from "@maple/backend/platform/describe-cause" import { Cause, Clock, Duration, Effect } from "effect" import type { AgentCompletion, AgentDefinition } from "./agents" import type { AutonomousOutcome, PhaseKind } from "./autonomous-outcome" +import { CLOSE_OUT_MAX_DURATION } from "./budgets" import { withToolTranscript } from "./close-out" import { reviewFailureReason } from "./review/failure" import type { ChatRunOutcome } from "./run" @@ -18,6 +19,18 @@ import type { ChatRunOutcome } from "./run" /** A phase with less than this much of the pass's clock left is not started. */ const MIN_CONTINUE_MS = 2 * 60 * 1000 +/** The session's agent on the close-out's own clock, so a long pass cannot push the turn past its watchdog. */ +export const closeOutAgent = (agent: AgentDefinition): AgentDefinition => ({ + ...agent, + budget: { + ...agent.budget, + maxDuration: Duration.min( + Duration.fromInputUnsafe(agent.budget.maxDuration), + Duration.fromInputUnsafe(CLOSE_OUT_MAX_DURATION), + ), + }, +}) + /** One run of the turn. */ export interface PhaseTurn { readonly text: string @@ -125,6 +138,7 @@ export const runAutonomousPhases = (input: AutonomousPhasesInput): E text: completion.closeOutPrompt, history: withToolTranscript(input.transcript()), closeOut: true, + agent: closeOutAgent(agent), }) yield* Effect.annotateCurrentSpan(completion.closedOutAttribute, closeOut.submitted) } diff --git a/apps/ai/src/chat/budgets.test.ts b/apps/ai/src/chat/budgets.test.ts index 995e25f76d..a8f053df27 100644 --- a/apps/ai/src/chat/budgets.test.ts +++ b/apps/ai/src/chat/budgets.test.ts @@ -18,6 +18,7 @@ import { PR_REVIEW_BUDGET, type AgentBudget, } from "./budgets" +import { closeOutAgent } from "./autonomous-phases" import { TURN_STALE_MS } from "./ChatSession" /** The window `z-ai/glm-5.3-flash:nitro` reports, which is what made this a bug. */ @@ -85,9 +86,12 @@ describe("agent budgets", () => { const ms = (budget: AgentBudget) => Duration.toMillis(budget.maxDuration) const margin = 5 * 60 * 1000 expect(ms(CHAT_BUDGET) + margin).toBeLessThanOrEqual(TURN_STALE_MS) - // An unattended pass that ends without its report gets a close-out run under the same budget. + // An unattended pass that ends without its report gets a close-out run on its own clock. for (const budget of [INVESTIGATION_BUDGET, PR_REVIEW_BUDGET, PR_REPLY_BUDGET]) { - expect(2 * ms(budget) + margin).toBeLessThanOrEqual(TURN_STALE_MS) + const closeOut = Duration.toMillis( + closeOutAgent({ ...AGENTS.investigate, budget }).budget.maxDuration, + ) + expect(ms(budget) + closeOut + margin).toBeLessThanOrEqual(TURN_STALE_MS) } }) diff --git a/apps/ai/src/chat/budgets.ts b/apps/ai/src/chat/budgets.ts index 66a4e8555b..cf1fba2b45 100644 --- a/apps/ai/src/chat/budgets.ts +++ b/apps/ai/src/chat/budgets.ts @@ -82,10 +82,14 @@ export interface AgentBudget { * * Runaway guards only, far above the 54 calls the worst observed run made; the wall clock is what * ends a stuck pass. A run cost about $0.04 at 79% cache reads (2026-09-19), so a long one is cheap. + * + * 15 minutes: at 10, 17% of passes ran out of clock (2026-11-11..16) after 9-17 model calls that + * spent 90% of the pass streaming at about 45 tokens a second. The 80% warning at 10 minutes left + * room for one such call; at 12 it leaves room for two. */ export const INVESTIGATION_BUDGET: AgentBudget = { maxToolCalls: 200, - maxDuration: "10 minutes", + maxDuration: "15 minutes", tokenBudget: 25_600_000, completionReserveTokens: 64_000, // Without its clock the pass ran into the 10-minute rail instead of submitting before it. @@ -101,13 +105,19 @@ export const INVESTIGATION_BUDGET: AgentBudget = { */ export const PR_REVIEW_BUDGET: AgentBudget = { maxToolCalls: 500, - // Twice this plus the margin must stay under `TURN_STALE_MS`: a pass and its close-out. + // This plus `CLOSE_OUT_MAX_DURATION` and the margin must stay under `TURN_STALE_MS`. maxDuration: "10 minutes", tokenBudget: 64_000_000, completionReserveTokens: 48_000, runStatus: true, } +/** + * The most a close-out run may take, whatever the pass had. One or two calls from the transcript + * (56-204s observed); the cap is what keeps a pass and its close-out under `TURN_STALE_MS`. + */ +export const CLOSE_OUT_MAX_DURATION: Duration.Input = "5 minutes" + /** An answer on a pull request: narrower than a review, with room to read and to stage a fix. */ export const PR_REPLY_BUDGET: AgentBudget = { maxToolCalls: 200, diff --git a/apps/ai/src/chat/close-out.test.ts b/apps/ai/src/chat/close-out.test.ts index 8601ff04c7..acaf66d120 100644 --- a/apps/ai/src/chat/close-out.test.ts +++ b/apps/ai/src/chat/close-out.test.ts @@ -3,6 +3,7 @@ */ import { ChatMessage, ChatToolCall } from "@maple/domain/chat-session" import { assert, describe, it } from "vitest" +import { MAX_LIVE_CONTEXT_TOKENS } from "./budgets" import { CLOSE_OUT_EVIDENCE_CHARS, withToolTranscript } from "./close-out" import { String as Str } from "effect" @@ -69,4 +70,17 @@ describe("withToolTranscript", () => { assert.notInclude(text, "src/f0.ts") assert.match(text, /\(\d+ earlier tool calls left out for length\.\)/) }) + + /** + * An investigation close-out opened at 123k of its 128k limit with a 39k system prompt, and the + * round after its submission failed with `ContextBudgetError` (2026-11-16). + */ + it("leaves a close-out room for the round after it submits", () => { + const SYSTEM_PROMPT_TOKENS = 39_282 + const CHARS_PER_TOKEN = 2.4 + const SUBMIT_ROUND_TOKENS = 20_000 + const [message] = withToolTranscript([pass(149, 19_000)]) + const evidenceTokens = (message?.text.length ?? Infinity) / CHARS_PER_TOKEN + assert.isAtMost(SYSTEM_PROMPT_TOKENS + evidenceTokens + SUBMIT_ROUND_TOKENS, MAX_LIVE_CONTEXT_TOKENS) + }) }) diff --git a/apps/ai/src/chat/close-out.ts b/apps/ai/src/chat/close-out.ts index 067342a5ca..59c2c4c0c2 100644 --- a/apps/ai/src/chat/close-out.ts +++ b/apps/ai/src/chat/close-out.ts @@ -5,10 +5,14 @@ import { ChatMessage } from "@maple/domain/chat-session" const CLOSE_OUT_TOOL_OUTPUT_CHARS = 20_000 /** - * All the evidence the close-out is shown, about 60k tokens. The newest message is replayed whole + * All the evidence the close-out is shown, about 50k tokens. The newest message is replayed whole * and compaction cannot split it, so an unbounded transcript fails the close-out before its first call. + * + * At 200k an investigation's close-out opened at 123k tokens (tool output runs about 2.4 characters + * a token, over a 39k system prompt), and the round after `submit_diagnosis` no longer fit the + * 128k limit (2026-11-16). This leaves that round room. */ -export const CLOSE_OUT_EVIDENCE_CHARS = 200_000 +export const CLOSE_OUT_EVIDENCE_CHARS = 120_000 /** * The transcript as the close-out sees it: the same messages, with each assistant message's tool diff --git a/apps/ai/src/platform/genai-spans.ts b/apps/ai/src/platform/genai-spans.ts index db03fa5e3f..2bf446796e 100644 --- a/apps/ai/src/platform/genai-spans.ts +++ b/apps/ai/src/platform/genai-spans.ts @@ -23,6 +23,7 @@ import { Duration, Effect, Exit, + Fiber, Layer, Option, Predicate, @@ -388,13 +389,13 @@ const callTimeout = (): AiError.AiError => streamTimeout(`Model stream did not finish within ${MODEL_CALL_TIMEOUT}`) /** - * How an unattended model call is attempted: the first attempt is cut at `cap` and run again with - * `retry` applied, on whatever is left of {@link MODEL_CALL_TIMEOUT}. + * How an unattended model call is attempted: past `cap` the call is asked again with `retry` + * applied while the first attempt keeps going, on whatever is left of {@link MODEL_CALL_TIMEOUT}. * * A slow call is a long generation, not a stalled one: the 190-231s calls of 2026-10-15 streamed * 5-6k reasoning tokens at about 34 per second, first chunk at 14s. Only a retry that reasons less - * returns sooner, which is why `retry` exists. The first attempt is held until it finishes, because - * the engine never repeats a call that streamed content; that costs nothing but live streaming. + * returns sooner, which is why `retry` exists. Each attempt is held until it finishes, because the + * engine never repeats a call that streamed content; that costs nothing but live streaming. */ export interface ModelCallAttempts { readonly cap: Duration.Duration @@ -421,7 +422,7 @@ export const instrumentLanguageModel = ( instrumentation: { /** Join streamed deltas (see `coalesceDeltas`); for models only unattended runs use. */ readonly coalesceDeltas?: boolean - /** Cut and retry a long call (see {@link ModelCallAttempts}); for unattended runs only. */ + /** Race a retry against a long call (see {@link ModelCallAttempts}); for unattended runs only. */ readonly attempts?: ModelCallAttempts } = {}, ): Effect.Effect => @@ -492,25 +493,40 @@ export const instrumentLanguageModel = ( }), ) -/** The first attempt, replayed once it finishes inside the cap; otherwise the retry, streamed live. */ +/** + * The first attempt, replayed once it finishes inside the cap. Past the cap the retry starts beside + * it and whichever answers first is replayed: cutting the first attempt threw away its 90s, and the + * low-effort retry of an investigation took 83s more at the median (2026-11-16). + */ const retriedPastCap = ( attempt: (options: O, ordinal: number) => Stream.Stream, options: O, attempts: ModelCallAttempts, ): Stream.Stream => Stream.unwrap( - Stream.runCollect(attempt(options, 1)).pipe( - Effect.timeoutOption(attempts.cap), - Effect.flatMap( - Option.match({ - onSome: (parts) => Effect.succeed(Stream.fromIterable(parts)), - onNone: () => - Effect.logWarning("Model call cut at its attempt cap; retrying").pipe( - Effect.annotateLogs({ capMillis: Duration.toMillis(attempts.cap) }), - Effect.as(attempts.retry(attempt(options, 2))), - ), - }), + Effect.forkChild(Stream.runCollect(attempt(options, 1))).pipe( + Effect.flatMap((first) => + Fiber.join(first).pipe( + Effect.timeoutOption(attempts.cap), + Effect.flatMap( + Option.match({ + onSome: Effect.succeed, + onNone: () => + Effect.logWarning("Model call past its attempt cap; racing a retry").pipe( + Effect.annotateLogs({ capMillis: Duration.toMillis(attempts.cap) }), + Effect.andThen( + Effect.race( + Fiber.join(first), + Stream.runCollect(attempts.retry(attempt(options, 2))), + ), + ), + ), + }), + ), + Effect.ensuring(Fiber.interrupt(first)), + ), ), + Effect.map((parts) => Stream.fromIterable(parts)), ), ) diff --git a/apps/ai/src/platform/model-call-span.test.ts b/apps/ai/src/platform/model-call-span.test.ts index 36ec45961f..91518e6427 100644 --- a/apps/ai/src/platform/model-call-span.test.ts +++ b/apps/ai/src/platform/model-call-span.test.ts @@ -837,6 +837,36 @@ describe("model calls cut at their attempt cap", () => { }), ) + it.effect("keeps the first attempt past the cap and delivers it when it beats the retry", () => + Effect.gen(function* () { + const retried: Array = [] + // Reasons for 100s, then answers. + const slowAnswer = slowReasoning.pipe(Stream.take(11), Stream.concat(quickAnswer)) + const fiber = yield* LanguageModel.streamText({ prompt: "hi" }).pipe( + Stream.runCollect, + Effect.provide(scripted([slowAnswer, slowReasoning], retried)), + Effect.forkChild, + ) + // A second at a time, so each of the first attempt's chained sleeps gets to run. + const seconds = (n: number) => + Effect.forEach(Arr.range(1, n), () => TestClock.adjust("1 second"), { discard: true }) + yield* seconds(91) + assert.lengthOf(retried, 1, "the retry starts at the cap") + assert.isUndefined(fiber.pollUnsafe()) + // The retry never answers, so only the first attempt can. + yield* seconds(15) + const parts = yield* Fiber.join(fiber) + assert.deepStrictEqual( + Arr.map(Arr.takeRight(parts, 4), (part) => part.type), + ["text-start", "text-delta", "text-end", "finish"], + ) + assert.isTrue( + Arr.some(parts, (part) => part.type === "reasoning-delta"), + "the first attempt's own parts", + ) + }), + ) + it.effect("delivers a call that finishes inside the cap as it was, without a retry", () => Effect.gen(function* () { const retried: Array = [] From a7978bb8515bcd7c9335543a508c0448b22c61fc Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 16 Nov 2026 17:30:47 +0100 Subject: [PATCH 2/2] fix(ai): a failed first answer past the cap waits for the retry --- apps/ai/src/platform/genai-spans.ts | 31 ++++++++++++++-- apps/ai/src/platform/model-call-span.test.ts | 38 ++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/apps/ai/src/platform/genai-spans.ts b/apps/ai/src/platform/genai-spans.ts index 2bf446796e..747f98babb 100644 --- a/apps/ai/src/platform/genai-spans.ts +++ b/apps/ai/src/platform/genai-spans.ts @@ -498,6 +498,17 @@ export const instrumentLanguageModel = ( * it and whichever answers first is replayed: cutting the first attempt threw away its 90s, and the * low-effort retry of an investigation took 83s more at the median (2026-11-16). */ +/** A collected response whose finish part reports a provider failure. */ +const answeredWithError = (parts: ReadonlyArray): boolean => + Arr.some( + parts, + (part) => + Predicate.hasProperty(part, "type") && + part.type === "finish" && + Predicate.hasProperty(part, "reason") && + part.reason === "error", + ) + const retriedPastCap = ( attempt: (options: O, ordinal: number) => Stream.Stream, options: O, @@ -515,9 +526,25 @@ const retriedPastCap = ( Effect.logWarning("Model call past its attempt cap; racing a retry").pipe( Effect.annotateLogs({ capMillis: Duration.toMillis(attempts.cap) }), Effect.andThen( - Effect.race( - Fiber.join(first), + Effect.forkChild( Stream.runCollect(attempts.retry(attempt(options, 2))), + ).pipe( + Effect.flatMap((retry) => + Effect.race( + // A first attempt that answers with a failure waits for the retry, + // and is only replayed when the retry fails too. + Fiber.join(first).pipe( + Effect.filterOrElse( + (parts) => !answeredWithError(parts), + (parts) => + Fiber.join(retry).pipe( + Effect.orElseSucceed(() => parts), + ), + ), + ), + Fiber.join(retry), + ).pipe(Effect.ensuring(Fiber.interrupt(retry))), + ), ), ), ), diff --git a/apps/ai/src/platform/model-call-span.test.ts b/apps/ai/src/platform/model-call-span.test.ts index 91518e6427..c2bf8b6d1d 100644 --- a/apps/ai/src/platform/model-call-span.test.ts +++ b/apps/ai/src/platform/model-call-span.test.ts @@ -837,6 +837,44 @@ describe("model calls cut at their attempt cap", () => { }), ) + it.effect("waits for the retry when the first attempt answers with a failure past the cap", () => + Effect.gen(function* () { + const retried: Array = [] + const failedAnswer = slowReasoning.pipe( + Stream.take(11), + Stream.concat( + Stream.fromIterable([ + { + type: "finish", + reason: "error", + usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } }, + }, + ]), + ), + ) + const lateAnswer = Stream.fromEffect(Effect.sleep("30 seconds")).pipe( + Stream.drain, + Stream.concat(quickAnswer), + ) + const fiber = yield* LanguageModel.streamText({ prompt: "hi" }).pipe( + Stream.runCollect, + Effect.provide(scripted([failedAnswer, lateAnswer], retried)), + Effect.forkChild, + ) + const seconds = (n: number) => + Effect.forEach(Arr.range(1, n), () => TestClock.adjust("1 second"), { discard: true }) + yield* seconds(105) + assert.isUndefined(fiber.pollUnsafe(), "the failed first answer does not end the call") + yield* seconds(20) + const parts = yield* Fiber.join(fiber) + assert.isTrue( + Arr.some(parts, (part) => part.type === "text-delta"), + "the retry's answer", + ) + assert.isFalse(Arr.some(parts, (part) => part.type === "reasoning-delta")) + }), + ) + it.effect("keeps the first attempt past the cap and delivers it when it beats the retry", () => Effect.gen(function* () { const retried: Array = []