Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion apps/ai/src/chat/autonomous-phases.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { Effect, Array as Arr } from "effect"
import { Duration, Effect, Array as Arr } from "effect"
import { assert, describe, it } from "vitest"
import { AGENTS, type RowMode } from "./agents"
import { makeAutonomousOutcome } from "./autonomous-outcome"
import { CLOSE_OUT_MAX_DURATION } from "./budgets"
import { type PhaseTurn, runAutonomousPhases } from "./autonomous-phases"

/** Runs the phases over a scripted run: each phase files when `files` says so, and ends on `stop`. */
Expand Down Expand Up @@ -57,6 +58,20 @@ describe("runAutonomousPhases", () => {
assert.isFalse(outcome.submitted())
})

it("runs an investigation's close-out on its own short clock, not the pass's", async () => {
const { turns } = await phasesOf("investigate", (turn) => turn.closeOut === true)
const clock = (turn: PhaseTurn | undefined) =>
turn?.agent === undefined ? undefined : Duration.toMillis(turn.agent.budget.maxDuration)
assert.strictEqual(
clock(turns[1]),
Duration.toMillis(Duration.fromInputUnsafe(CLOSE_OUT_MAX_DURATION)),
)
assert.isBelow(
clock(turns[1]) ?? Infinity,
Duration.toMillis(Duration.fromInputUnsafe(AGENTS.investigate.budget.maxDuration)),
)
})

it("closes a reply out without a continuation, which only a review has", async () => {
const { turns } = await phasesOf("pr-reply", () => false)
assert.deepEqual(
Expand Down
14 changes: 14 additions & 0 deletions apps/ai/src/chat/autonomous-phases.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,26 @@ import { summarizeCause } from "@maple/backend/platform/describe-cause"
import { Cause, Clock, Duration, Effect } from "effect"
import type { AgentCompletion, AgentDefinition } from "./agents"
import type { AutonomousOutcome, PhaseKind } from "./autonomous-outcome"
import { CLOSE_OUT_MAX_DURATION } from "./budgets"
import { withToolTranscript } from "./close-out"
import { reviewFailureReason } from "./review/failure"
import type { ChatRunOutcome } from "./run"

/** A phase with less than this much of the pass's clock left is not started. */
const MIN_CONTINUE_MS = 2 * 60 * 1000

/** The session's agent on the close-out's own clock, so a long pass cannot push the turn past its watchdog. */
export const closeOutAgent = (agent: AgentDefinition): AgentDefinition => ({
...agent,
budget: {
...agent.budget,
maxDuration: Duration.min(
Duration.fromInputUnsafe(agent.budget.maxDuration),
Duration.fromInputUnsafe(CLOSE_OUT_MAX_DURATION),
),
},
})

/** One run of the turn. */
export interface PhaseTurn {
readonly text: string
Expand Down Expand Up @@ -125,6 +138,7 @@ export const runAutonomousPhases = <E, R>(input: AutonomousPhasesInput<E, R>): E
text: completion.closeOutPrompt,
history: withToolTranscript(input.transcript()),
closeOut: true,
agent: closeOutAgent(agent),
})
yield* Effect.annotateCurrentSpan(completion.closedOutAttribute, closeOut.submitted)
}
Expand Down
8 changes: 6 additions & 2 deletions apps/ai/src/chat/budgets.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
PR_REVIEW_BUDGET,
type AgentBudget,
} from "./budgets"
import { closeOutAgent } from "./autonomous-phases"
import { TURN_STALE_MS } from "./ChatSession"

/** The window `z-ai/glm-5.3-flash:nitro` reports, which is what made this a bug. */
Expand Down Expand Up @@ -85,9 +86,12 @@ describe("agent budgets", () => {
const ms = (budget: AgentBudget) => Duration.toMillis(budget.maxDuration)
const margin = 5 * 60 * 1000
expect(ms(CHAT_BUDGET) + margin).toBeLessThanOrEqual(TURN_STALE_MS)
// An unattended pass that ends without its report gets a close-out run under the same budget.
// An unattended pass that ends without its report gets a close-out run on its own clock.
for (const budget of [INVESTIGATION_BUDGET, PR_REVIEW_BUDGET, PR_REPLY_BUDGET]) {
expect(2 * ms(budget) + margin).toBeLessThanOrEqual(TURN_STALE_MS)
const closeOut = Duration.toMillis(
closeOutAgent({ ...AGENTS.investigate, budget }).budget.maxDuration,
)
expect(ms(budget) + closeOut + margin).toBeLessThanOrEqual(TURN_STALE_MS)
}
})

Expand Down
14 changes: 12 additions & 2 deletions apps/ai/src/chat/budgets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,14 @@ export interface AgentBudget {
*
* Runaway guards only, far above the 54 calls the worst observed run made; the wall clock is what
* ends a stuck pass. A run cost about $0.04 at 79% cache reads (2026-09-19), so a long one is cheap.
*
* 15 minutes: at 10, 17% of passes ran out of clock (2026-11-11..16) after 9-17 model calls that
* spent 90% of the pass streaming at about 45 tokens a second. The 80% warning at 10 minutes left
* room for one such call; at 12 it leaves room for two.
*/
export const INVESTIGATION_BUDGET: AgentBudget = {
maxToolCalls: 200,
maxDuration: "10 minutes",
maxDuration: "15 minutes",
tokenBudget: 25_600_000,
completionReserveTokens: 64_000,
// Without its clock the pass ran into the 10-minute rail instead of submitting before it.
Expand All @@ -101,13 +105,19 @@ export const INVESTIGATION_BUDGET: AgentBudget = {
*/
export const PR_REVIEW_BUDGET: AgentBudget = {
maxToolCalls: 500,
// Twice this plus the margin must stay under `TURN_STALE_MS`: a pass and its close-out.
// This plus `CLOSE_OUT_MAX_DURATION` and the margin must stay under `TURN_STALE_MS`.
maxDuration: "10 minutes",
tokenBudget: 64_000_000,
completionReserveTokens: 48_000,
runStatus: true,
}

/**
* The most a close-out run may take, whatever the pass had. One or two calls from the transcript
* (56-204s observed); the cap is what keeps a pass and its close-out under `TURN_STALE_MS`.
*/
export const CLOSE_OUT_MAX_DURATION: Duration.Input = "5 minutes"

/** An answer on a pull request: narrower than a review, with room to read and to stage a fix. */
export const PR_REPLY_BUDGET: AgentBudget = {
maxToolCalls: 200,
Expand Down
14 changes: 14 additions & 0 deletions apps/ai/src/chat/close-out.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
*/
import { ChatMessage, ChatToolCall } from "@maple/domain/chat-session"
import { assert, describe, it } from "vitest"
import { MAX_LIVE_CONTEXT_TOKENS } from "./budgets"
import { CLOSE_OUT_EVIDENCE_CHARS, withToolTranscript } from "./close-out"
import { String as Str } from "effect"

Expand Down Expand Up @@ -69,4 +70,17 @@ describe("withToolTranscript", () => {
assert.notInclude(text, "src/f0.ts")
assert.match(text, /\(\d+ earlier tool calls left out for length\.\)/)
})

/**
* An investigation close-out opened at 123k of its 128k limit with a 39k system prompt, and the
* round after its submission failed with `ContextBudgetError` (2026-11-16).
*/
it("leaves a close-out room for the round after it submits", () => {
const SYSTEM_PROMPT_TOKENS = 39_282
const CHARS_PER_TOKEN = 2.4
const SUBMIT_ROUND_TOKENS = 20_000
const [message] = withToolTranscript([pass(149, 19_000)])
const evidenceTokens = (message?.text.length ?? Infinity) / CHARS_PER_TOKEN
assert.isAtMost(SYSTEM_PROMPT_TOKENS + evidenceTokens + SUBMIT_ROUND_TOKENS, MAX_LIVE_CONTEXT_TOKENS)
})
})
8 changes: 6 additions & 2 deletions apps/ai/src/chat/close-out.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@ import { ChatMessage } from "@maple/domain/chat-session"
const CLOSE_OUT_TOOL_OUTPUT_CHARS = 20_000

/**
* All the evidence the close-out is shown, about 60k tokens. The newest message is replayed whole
* All the evidence the close-out is shown, about 50k tokens. The newest message is replayed whole
* and compaction cannot split it, so an unbounded transcript fails the close-out before its first call.
*
* At 200k an investigation's close-out opened at 123k tokens (tool output runs about 2.4 characters
* a token, over a 39k system prompt), and the round after `submit_diagnosis` no longer fit the
* 128k limit (2026-11-16). This leaves that round room.
*/
export const CLOSE_OUT_EVIDENCE_CHARS = 200_000
export const CLOSE_OUT_EVIDENCE_CHARS = 120_000

/**
* The transcript as the close-out sees it: the same messages, with each assistant message's tool
Expand Down
77 changes: 60 additions & 17 deletions apps/ai/src/platform/genai-spans.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
Duration,
Effect,
Exit,
Fiber,
Layer,
Option,
Predicate,
Expand Down Expand Up @@ -388,13 +389,13 @@ const callTimeout = (): AiError.AiError =>
streamTimeout(`Model stream did not finish within ${MODEL_CALL_TIMEOUT}`)

/**
* How an unattended model call is attempted: the first attempt is cut at `cap` and run again with
* `retry` applied, on whatever is left of {@link MODEL_CALL_TIMEOUT}.
* How an unattended model call is attempted: past `cap` the call is asked again with `retry`
* applied while the first attempt keeps going, on whatever is left of {@link MODEL_CALL_TIMEOUT}.
*
* A slow call is a long generation, not a stalled one: the 190-231s calls of 2026-10-15 streamed
* 5-6k reasoning tokens at about 34 per second, first chunk at 14s. Only a retry that reasons less
* returns sooner, which is why `retry` exists. The first attempt is held until it finishes, because
* the engine never repeats a call that streamed content; that costs nothing but live streaming.
* returns sooner, which is why `retry` exists. Each attempt is held until it finishes, because the
* engine never repeats a call that streamed content; that costs nothing but live streaming.
*/
export interface ModelCallAttempts {
readonly cap: Duration.Duration
Expand All @@ -421,7 +422,7 @@ export const instrumentLanguageModel = <R>(
instrumentation: {
/** Join streamed deltas (see `coalesceDeltas`); for models only unattended runs use. */
readonly coalesceDeltas?: boolean
/** Cut and retry a long call (see {@link ModelCallAttempts}); for unattended runs only. */
/** Race a retry against a long call (see {@link ModelCallAttempts}); for unattended runs only. */
readonly attempts?: ModelCallAttempts
} = {},
): Effect.Effect<LanguageModel.LanguageModel, never, R> =>
Expand Down Expand Up @@ -492,25 +493,67 @@ export const instrumentLanguageModel = <R>(
}),
)

/** The first attempt, replayed once it finishes inside the cap; otherwise the retry, streamed live. */
/**
* The first attempt, replayed once it finishes inside the cap. Past the cap the retry starts beside
* it and whichever answers first is replayed: cutting the first attempt threw away its 90s, and the
* low-effort retry of an investigation took 83s more at the median (2026-11-16).
*/
/** A collected response whose finish part reports a provider failure. */
const answeredWithError = (parts: ReadonlyArray<unknown>): boolean =>
Arr.some(
parts,
(part) =>
Predicate.hasProperty(part, "type") &&
part.type === "finish" &&
Predicate.hasProperty(part, "reason") &&
part.reason === "error",
)

const retriedPastCap = <O, A, E, R>(
attempt: (options: O, ordinal: number) => Stream.Stream<A, E, R>,
options: O,
attempts: ModelCallAttempts,
): Stream.Stream<A, E, R> =>
Stream.unwrap(
Stream.runCollect(attempt(options, 1)).pipe(
Effect.timeoutOption(attempts.cap),
Effect.flatMap(
Option.match({
onSome: (parts) => Effect.succeed(Stream.fromIterable(parts)),
onNone: () =>
Effect.logWarning("Model call cut at its attempt cap; retrying").pipe(
Effect.annotateLogs({ capMillis: Duration.toMillis(attempts.cap) }),
Effect.as(attempts.retry(attempt(options, 2))),
),
}),
Effect.forkChild(Stream.runCollect(attempt(options, 1))).pipe(
Effect.flatMap((first) =>
Fiber.join(first).pipe(
Effect.timeoutOption(attempts.cap),
Effect.flatMap(
Option.match({
onSome: Effect.succeed,
onNone: () =>
Effect.logWarning("Model call past its attempt cap; racing a retry").pipe(
Effect.annotateLogs({ capMillis: Duration.toMillis(attempts.cap) }),
Effect.andThen(
Effect.forkChild(
Stream.runCollect(attempts.retry(attempt(options, 2))),
).pipe(
Effect.flatMap((retry) =>
Effect.race(
// A first attempt that answers with a failure waits for the retry,
// and is only replayed when the retry fails too.
Fiber.join(first).pipe(
Effect.filterOrElse(
(parts) => !answeredWithError(parts),
(parts) =>
Fiber.join(retry).pipe(
Effect.orElseSucceed(() => parts),
),
),
),
Fiber.join(retry),
).pipe(Effect.ensuring(Fiber.interrupt(retry))),
),
),
),
),
}),
),
Effect.ensuring(Fiber.interrupt(first)),
),
),
Effect.map((parts) => Stream.fromIterable(parts)),
),
)

Expand Down
68 changes: 68 additions & 0 deletions apps/ai/src/platform/model-call-span.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,74 @@ describe("model calls cut at their attempt cap", () => {
}),
)

it.effect("waits for the retry when the first attempt answers with a failure past the cap", () =>
Effect.gen(function* () {
const retried: Array<number> = []
const failedAnswer = slowReasoning.pipe(
Stream.take(11),
Stream.concat(
Stream.fromIterable<AiResponse.StreamPartEncoded>([
{
type: "finish",
reason: "error",
usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } },
},
]),
),
)
const lateAnswer = Stream.fromEffect(Effect.sleep("30 seconds")).pipe(
Stream.drain,
Stream.concat(quickAnswer),
)
const fiber = yield* LanguageModel.streamText({ prompt: "hi" }).pipe(
Stream.runCollect,
Effect.provide(scripted([failedAnswer, lateAnswer], retried)),
Effect.forkChild,
)
const seconds = (n: number) =>
Effect.forEach(Arr.range(1, n), () => TestClock.adjust("1 second"), { discard: true })
yield* seconds(105)
assert.isUndefined(fiber.pollUnsafe(), "the failed first answer does not end the call")
yield* seconds(20)
const parts = yield* Fiber.join(fiber)
assert.isTrue(
Arr.some(parts, (part) => part.type === "text-delta"),
"the retry's answer",
)
assert.isFalse(Arr.some(parts, (part) => part.type === "reasoning-delta"))
}),
)

it.effect("keeps the first attempt past the cap and delivers it when it beats the retry", () =>
Effect.gen(function* () {
const retried: Array<number> = []
// Reasons for 100s, then answers.
const slowAnswer = slowReasoning.pipe(Stream.take(11), Stream.concat(quickAnswer))
const fiber = yield* LanguageModel.streamText({ prompt: "hi" }).pipe(
Stream.runCollect,
Effect.provide(scripted([slowAnswer, slowReasoning], retried)),
Effect.forkChild,
)
// A second at a time, so each of the first attempt's chained sleeps gets to run.
const seconds = (n: number) =>
Effect.forEach(Arr.range(1, n), () => TestClock.adjust("1 second"), { discard: true })
yield* seconds(91)
assert.lengthOf(retried, 1, "the retry starts at the cap")
assert.isUndefined(fiber.pollUnsafe())
// The retry never answers, so only the first attempt can.
yield* seconds(15)
const parts = yield* Fiber.join(fiber)
assert.deepStrictEqual(
Arr.map(Arr.takeRight(parts, 4), (part) => part.type),
["text-start", "text-delta", "text-end", "finish"],
)
assert.isTrue(
Arr.some(parts, (part) => part.type === "reasoning-delta"),
"the first attempt's own parts",
)
}),
)

it.effect("delivers a call that finishes inside the cap as it was, without a retry", () =>
Effect.gen(function* () {
const retried: Array<number> = []
Expand Down
Loading