From 91198cc0b6ea5de9939e66311f036c24eb3fe79c Mon Sep 17 00:00:00 2001 From: liuedcson <332840681+liuedcson@users.noreply.github.com> Date: Sat, 21 Nov 2026 11:22:48 +0800 Subject: [PATCH 1/2] feat(search): fall back to credential-free providers for web search `search_web` required a StepFun search credential, so a session using a self-configured model could not search at all: it failed with "requires a credential", or used a stale key against the StepFun endpoint and received 401/402. Search was also bound at tool creation time, so logging in during a session had no effect. Resolve the backend on every call and walk an ordered candidate chain: - The StepFun endpoint stays first whenever the current login resolves a search credential, which keeps paid users on the better path. A quota, auth, network or timeout failure falls through to the free providers instead of failing the tool. - Without a credential the paid endpoint is absent from the chain rather than attempted and failed, which skips a call that cannot succeed. - The free providers (Exa, Firecrawl, Tavily, Parallel, TinyFish) are called without a credential, in a per-call shuffled order so traffic is spread across their independent free allowances. - Selection is re-read per call from STEPCODE_SEARCH_PROVIDER, then [search] provider, then auto. An unrecognized value degrades to auto. Failures are classified by the provider's `isError` flag first, then the status code, then the body text: several providers answer HTTP 200 with an error body, and treating that as "no results" would stop the chain and hide an exhausted quota. Each failure is attributed to its provider, cooldowns skip known-bad candidates, and a rejected credential-free provider is skipped for the rest of the process. The tool contract is unchanged: same name, same input, same rendering and details fields, same permission entry. --- docs/step-configuration.md | 24 ++ .../coding-agent/docs/step-integration.md | 55 ++- packages/coding-agent/src/step/config-toml.ts | 7 + .../coding-agent/src/step/search-web-tool.ts | 273 +++++++++++--- .../coding-agent/src/step/search/adapters.ts | 286 ++++++++++++++ .../coding-agent/src/step/search/errors.ts | 111 ++++++ .../coding-agent/src/step/search/routing.ts | 92 +++++ .../coding-agent/src/step/search/scheduler.ts | 202 ++++++++++ .../coding-agent/src/step/search/transport.ts | 288 ++++++++++++++ .../coding-agent/src/step/search/types.ts | 117 ++++++ .../test/fixtures/search/exa.json | 15 + .../test/fixtures/search/firecrawl.json | 44 +++ .../test/fixtures/search/parallel.json | 45 +++ .../test/fixtures/search/tavily.json | 30 ++ .../test/fixtures/search/tinyfish.json | 12 + .../test/step-search-chain.test.ts | 352 ++++++++++++++++++ .../test/step-search-config.test.ts | 64 ++++ .../test/step-search-scenarios.test.ts | 278 ++++++++++++++ .../test/step-search-transport.test.ts | 305 +++++++++++++++ .../coding-agent/test/step-search-web.test.ts | 187 ++++++++-- 20 files changed, 2677 insertions(+), 110 deletions(-) create mode 100644 packages/coding-agent/src/step/search/adapters.ts create mode 100644 packages/coding-agent/src/step/search/errors.ts create mode 100644 packages/coding-agent/src/step/search/routing.ts create mode 100644 packages/coding-agent/src/step/search/scheduler.ts create mode 100644 packages/coding-agent/src/step/search/transport.ts create mode 100644 packages/coding-agent/src/step/search/types.ts create mode 100644 packages/coding-agent/test/fixtures/search/exa.json create mode 100644 packages/coding-agent/test/fixtures/search/firecrawl.json create mode 100644 packages/coding-agent/test/fixtures/search/parallel.json create mode 100644 packages/coding-agent/test/fixtures/search/tavily.json create mode 100644 packages/coding-agent/test/fixtures/search/tinyfish.json create mode 100644 packages/coding-agent/test/step-search-chain.test.ts create mode 100644 packages/coding-agent/test/step-search-config.test.ts create mode 100644 packages/coding-agent/test/step-search-scenarios.test.ts create mode 100644 packages/coding-agent/test/step-search-transport.test.ts diff --git a/docs/step-configuration.md b/docs/step-configuration.md index d7182f488..ea501ae07 100644 --- a/docs/step-configuration.md +++ b/docs/step-configuration.md @@ -209,6 +209,30 @@ the prompt; deleting that line asks again. `/theme` changes the theme later, and launches that already carry work (an initial prompt, a resumed session) skip the prompt entirely. +## Web search backend selection + +`search_web` picks its backend from `[search] provider` in `config.toml`, or the +`STEPCODE_SEARCH_PROVIDER` environment variable, which takes precedence: + +```toml +[search] +provider = "auto" # auto | step | exa | firecrawl | parallel | tavily | tinyfish | off +``` + +`auto` walks an ordered chain. The Step search endpoint comes first when the +current login resolves a search credential; otherwise the credential-free +providers are used directly, without attempting the paid endpoint. A paid-tier +failure falls through to the free providers. A provider id pins that backend and +never falls back, and `off` keeps the tool registered but refuses the call. +Any other value is treated as `auto`, so an older configuration or a removed +provider never disables search. Selection is read on every call, so an edit +applies to the next search rather than requiring a new session. + +The free providers are called without a credential. They have no configured key +and do not accept a Step credential, so their free allowance is the ceiling. +Their order is shuffled per call, which spreads traffic across the per-provider +allowances rather than always spending the first provider's. + ## MCP startup in the terminal Interactive Step sessions start MCP discovery and connections in the background. diff --git a/packages/coding-agent/docs/step-integration.md b/packages/coding-agent/docs/step-integration.md index 6dc744cf6..15631be37 100644 --- a/packages/coding-agent/docs/step-integration.md +++ b/packages/coding-agent/docs/step-integration.md @@ -94,12 +94,15 @@ from it, and `step login status` validates the credential against that profile's binary updates and the steppage plugin still use mainland endpoints regardless of profile. -The Step tool profile also registers `search_web`, backed by the remote -`stepsearch.web_search` Streamable HTTP MCP tool. The search credential is -resolved from an explicit `step --api-key`, then `STEPCODE_SEARCH_API_KEY`, -then the Step login entry in `auth.json`; it is sent only as a Bearer header. -Each login profile has its own endpoint, because the endpoint decides which -account the search is billed to: `step_plan` uses +The Step tool profile also registers `search_web`, which resolves an ordered +candidate chain on every call instead of binding one endpoint when the session +starts. + +The paid tier is the remote `stepsearch.web_search` Streamable HTTP MCP tool. +Its credential is resolved from an explicit `step --api-key`, then +`STEPCODE_SEARCH_API_KEY`, then the Step login entry in `auth.json`; it is sent +only as a Bearer header. Each login profile has its own endpoint, because the +endpoint decides which account the search is billed to: `step_plan` uses `https://api.stepfun.com/step_plan/v1/mcp/web_search/mcp` and `step_plan_oversea` uses `https://api.stepfun.ai/step_plan/v1/mcp/web_search/mcp`, both billed to the Step Plan quota; `platform_cn` uses @@ -112,13 +115,37 @@ a Step Plan credential is never one of those, since it only comes from `/login`, which always records a profile. `STEPCODE_SEARCH_WEB_MCP_URL` overrides this profile-based selection; an override that supplies only an origin keeps the path of the profile's own endpoint, so redirecting the host cannot move a plan user's -searches onto the billed platform path. This adapter does not add a separate -search credential store or `integrations.search` settings surface. The -`STEP_API_KEY` environment variable is deliberately excluded from that -chain: StepCode injects it together with `STEP_BASE_URL` to reach its own -model gateway, and because the search endpoint never follows that base URL, -reusing the value would authenticate a gateway key against -`api.stepfun.com` and fail. +searches onto the billed platform path. + +When the credential resolves, the paid endpoint is the first candidate. When it +does not, the paid endpoint is not attempted at all: the free candidates are +used directly, so a call that could only fail is never made. A paid-tier failure +(quota exhausted, rejected credential, unreachable endpoint) advances to the +free candidates instead of failing the tool, and the answer records which +backend served it. + +The free candidates are Exa, Firecrawl, Tavily, Parallel, and TinyFish, all +called without a credential: these endpoints do not accept a Step credential and +none of them is configured with one. They answer a single stateless +`tools/call`, skipping the handshake and catalog listing the paid endpoint +performs; a provider that refuses the stateless form is retried once through the +MCP client. Their order is shuffled per call with a deterministic fallback, so +traffic is spread across per-provider free allowances instead of always spending +the first one's. A failed candidate is skipped for a cooldown instead of being +retried on the next call, the whole walk shares one 25-second budget, and a +caller cancellation stops it immediately. + +Selection comes from `STEPCODE_SEARCH_PROVIDER`, then `[search] provider` in +`config.toml`, then `auto`. `auto` walks the chain; a provider id pins that one +backend and never falls back; `off` keeps the tool registered but refuses the +call. Any other value is treated as `auto`, so an older configuration or a +removed provider never disables search; selection is read on every call. This +adapter does not add a separate search credential store or +`integrations.search` settings surface. The `STEP_API_KEY` environment variable +is deliberately excluded from that chain: StepCode injects it together with +`STEP_BASE_URL` to reach its own model gateway, and because the search endpoint +never follows that base URL, reusing the value would authenticate a gateway key +against `api.stepfun.com` and fail. Session storage is also selected through the Step wrapper. The wrapper keeps the native `SessionManager` class, JSONL format, and tree operations unchanged, @@ -457,7 +484,7 @@ service and are not silently faked by this fork. | StepCode OAuth login/logout and legacy credential normalization | `features/step-provider`, `step/auth.ts`, `step/models-endpoint-repair.ts` | aligned; endpoint repair is tested | | Global/project settings and session paths | Native managers wrapped by `step/settings-manager.ts` and `step/session.ts` | aligned; writes use `.stepcode` | | Step permissions, `/permissions`, `/init` | Step extension/facades over native selectors and hooks | aligned; `/effort` replaces the native `/thinking`, while `/permission` and `/mode` remain absent | -| Built-in `search_web` | Step tool profile backed by `stepsearch.web_search` over Streamable HTTP MCP | aligned; uses search-specific environment overrides and the Step login credential fallback | +| Built-in `search_web` | Step tool profile backed by an ordered candidate chain: `stepsearch.web_search` over Streamable HTTP MCP, then credential-free providers | aligned; uses search-specific environment overrides, the Step login credential, and a shared failure budget | | Top-level `step feedback` and TUI `/feedback` | `step/feedback/`, `stepcode.ts`, and the Step extension over the current session/UI | aligned; the body and optional session bundle use separate Step collector requests | | `/status`, self-memory/skills governance, `/refresh`, `/rewind`, `/copy` full-transcript semantics | Previous gateway/TUI product layer | not yet ported; no misleading alias is registered | | `/multi-agent`, `/connect`, `/trace`, gateway/cron | Previous gateway and external channel services | separate product; requires an explicit Step service implementation | diff --git a/packages/coding-agent/src/step/config-toml.ts b/packages/coding-agent/src/step/config-toml.ts index e4737368b..d9d42e921 100644 --- a/packages/coding-agent/src/step/config-toml.ts +++ b/packages/coding-agent/src/step/config-toml.ts @@ -23,8 +23,15 @@ export interface StepMcpServerConfig { oauth?: { client_id?: string; client_secret?: string; scopes?: string[]; callback_port?: number }; } +/** Search backend selection; see `search-web-tool.ts` for the resolution order. */ +export interface StepSearchConfig { + /** `auto` walks the ordered candidate chain; a provider id pins one backend. */ + provider?: string; +} + export interface StepConfigDocument { mcp_servers?: Record; + search?: StepSearchConfig; [key: string]: unknown; } diff --git a/packages/coding-agent/src/step/search-web-tool.ts b/packages/coding-agent/src/step/search-web-tool.ts index 2bd196615..9711fc2d6 100644 --- a/packages/coding-agent/src/step/search-web-tool.ts +++ b/packages/coding-agent/src/step/search-web-tool.ts @@ -1,3 +1,4 @@ +import { existsSync } from "node:fs"; import { join } from "node:path"; import type { AgentToolResult } from "@step-harness/agent-core"; import type { Credential } from "@step-harness/providers"; @@ -5,10 +6,24 @@ import { type Static, Type } from "typebox"; import { readStoredCredential } from "../core/auth-storage.ts"; import type { ExtensionContext, ToolDefinition } from "../core/extensions/types.ts"; import { STEP_PROVIDER_ID } from "../features/step-provider/index.ts"; +import { readStepConfig, resolveStepConfigPath } from "./config-toml.ts"; import { resolveStepAgentDir } from "./environment.ts"; import { readStepLoginProfile } from "./login-flow.ts"; import { invokeRemoteMcpTool, type RemoteMcpToolInvocation, type RemoteMcpToolResult } from "./mcp-client.ts"; import type { StepLoginProfileId } from "./onboarding.ts"; +import { findAdapter } from "./search/adapters.ts"; +import { classifyFailure } from "./search/errors.ts"; +import { buildSearchChain } from "./search/routing.ts"; +import { runSearchChain, type SearchChainResult } from "./search/scheduler.ts"; +import { callSearchProvider } from "./search/transport.ts"; +import { + SEARCH_CHAIN_MODES, + type SearchAttempt, + type SearchChainMode, + type SearchFailure, + type SearchHit, + type SearchOutcome, +} from "./search/types.ts"; export { invokeRemoteMcpTool } from "./mcp-client.ts"; @@ -48,6 +63,9 @@ const SEARCH_WEB_PROFILE_URLS: Record = { const SEARCH_WEB_RESULT_COUNT = 10; const MAX_SNIPPET_CHARS = 400; +/** Environment override for the search backend selection. */ +export const SEARCH_PROVIDER_ENV = "STEPCODE_SEARCH_PROVIDER"; + const searchWebSchema = Type.Object({ query: Type.String({ description: "Search query text" }), }); @@ -63,6 +81,13 @@ export interface SearchWebToolOptions { authPath?: string; /** Injected environment, kept public so credential resolution is testable. */ env?: Record; + /** + * Backend selection. Defaults to `[search] provider` in config.toml, then + * `STEPCODE_SEARCH_PROVIDER`, then `auto`. + */ + provider?: SearchChainMode; + /** Injected randomness for the free-provider order; kept testable. */ + random?: () => number; } export type SearchWebInvocation = RemoteMcpToolInvocation & { @@ -75,6 +100,10 @@ export interface SearchWebDetails { query: string; resultCount: number; urls: string[]; + /** Backend that produced this answer. */ + provider?: string; + /** True when the answer came from a free provider instead of the paid tier. */ + degraded?: boolean; } export type SearchWebInvoker = (input: SearchWebInvocation) => Promise; @@ -140,15 +169,44 @@ export function buildSearchWebDescription(): string { ].join("\n\n"); } -/** Create the Step-facing tool backed by the remote web-search MCP server. */ +/** + * Resolve the configured backend. + * + * Selection is re-read on every call, so a configuration edit applies to the + * next search instead of requiring a new session. + * + * Supported values are `auto`, `off`, `step` and the five free provider ids. Any + * other value is treated as `auto`: the setting only ever names a preferred + * backend, so an unrecognized name (an older config, a provider that was + * removed) degrades to the default chain rather than disabling search. + */ +export function resolveSearchChainMode( + options: SearchWebToolOptions = {}, + env: Record = options.env ?? process.env, +): SearchChainMode { + const explicit = normalizeOptionalText(options.provider); + const fromEnv = normalizeOptionalText(env[SEARCH_PROVIDER_ENV]); + const fromConfig = readConfiguredProvider(env); + const requested = explicit ?? fromEnv ?? fromConfig; + if (!requested) return "auto"; + return (SEARCH_CHAIN_MODES as readonly string[]).includes(requested) ? (requested as SearchChainMode) : "auto"; +} + +function readConfiguredProvider(env: Record): string | undefined { + const path = resolveStepConfigPath(env); + if (!existsSync(path)) return undefined; + // A missing file contributes no setting; an unreadable or malformed one is a + // real configuration error and must not be reported as "no preference". + const document = readStepConfig(path); + return normalizeOptionalText(document.search?.provider); +} + +/** Create the Step-facing tool backed by the ordered search candidate chain. */ export function createSearchWebTool( options: SearchWebToolOptions = {}, invokeTool: SearchWebInvoker = invokeRemoteMcpTool, ): ToolDefinition { const env = options.env ?? process.env; - const authPath = resolveSearchWebAuthPath(options, env); - const serverUrl = resolveSearchWebServerUrl(options.url, env, readStepLoginProfile(authPath)); - const apiKey = resolveSearchWebApiKey(options); return { name: "search_web", @@ -161,87 +219,180 @@ export function createSearchWebTool( execute: async (_toolCallId, args: SearchWebInput, signal, _onUpdate, _ctx: ExtensionContext) => { const query = args.query.trim(); if (!query) throw new Error("search_web query must not be empty"); - if (!apiKey) { + + // Backend, credential and endpoint are resolved per call. A login or a + // configuration edit that happens during the session must be usable + // without restarting it, and a bad value must fail this call rather + // than the session that never used search. + const mode = resolveSearchChainMode(options, env); + const authPath = resolveSearchWebAuthPath(options, env); + const apiKey = resolveSearchWebApiKey(options); + const stepEndpoint = resolveSearchWebServerUrl(options.url, env, readStepLoginProfile(authPath)); + + const chain = buildSearchChain({ + mode, + hasStepCredential: Boolean(apiKey), + stepEndpoint, + ...(options.random ? { random: options.random } : {}), + }); + + if (chain.requiresCredential) { + // Pinned to the paid backend without a credential: a real + // configuration error, not a reason to silently use a free provider. throw new Error( "search_web requires a credential; run `/login` or `step login`, pass --api-key, or set STEPCODE_SEARCH_API_KEY", ); } - // The endpoint decides which account the search is billed to, so every - // failure names it: a profile/endpoint mismatch is otherwise invisible - // until it shows up on a bill. The URL carries no credential. - let result: SearchWebMcpResult; - try { - result = await invokeTool({ - serverName: SEARCH_WEB_SERVER_NAME, - serverUrl, - toolName: SEARCH_WEB_TOOL_NAME, - arguments: { query, n: SEARCH_WEB_RESULT_COUNT }, - headers: { Authorization: `Bearer ${apiKey}` }, - signal, - }); - } catch (error) { - throw new Error(`${error instanceof Error ? error.message : String(error)} (endpoint ${serverUrl})`); - } - if (result.isError) { - const detail = result.content || `MCP tool ${SEARCH_WEB_SERVER_NAME}.${SEARCH_WEB_TOOL_NAME} failed`; - throw new Error(`${detail} (endpoint ${serverUrl})`); - } + const result = await runSearchChain({ + chain, + query, + ...(signal ? { signal } : {}), + invoke: (candidate, searchQuery, attemptOptions) => { + if (candidate.tier === "stepfun") { + return invokeStepFun({ + invokeTool, + apiKey: apiKey ?? "", + candidate, + query: searchQuery, + timeoutMs: attemptOptions.timeoutMs, + signal: attemptOptions.signal, + }); + } + const adapter = findAdapter(candidate.id); + if (!adapter) { + return Promise.resolve({ + ok: false as const, + failure: { kind: "unknown" as const, detail: `unknown provider ${candidate.id}` }, + }); + } + return callSearchProvider({ + request: adapter.buildRequest(searchQuery), + parse: adapter.parse, + timeoutMs: attemptOptions.timeoutMs, + allowHandshake: candidate.allowHandshake, + serverName: candidate.id, + ...(attemptOptions.signal ? { signal: attemptOptions.signal } : {}), + }); + }, + }); + return renderSearchResults(query, result); }, }; } -function resolveSearchWebAuthPath(options: SearchWebToolOptions, env: Record): string { - return ( - normalizeOptionalText(options.authPath) ?? - normalizeOptionalText(env.STEPCODE_AUTH_PATH) ?? - join(resolveStepAgentDir(env), "auth.json") - ); -} - -function renderSearchResults(query: string, raw: SearchWebMcpResult): AgentToolResult { - const items = raw.structuredContent?.results; - if (!Array.isArray(items)) { - const content = raw.content?.trim() || `No search results for "${query}".`; +async function invokeStepFun(input: { + invokeTool: SearchWebInvoker; + apiKey: string; + candidate: { endpoint: string; toolName: string }; + query: string; + /** One attempt's slice of the shared budget; without it a hanging endpoint blocks the whole chain. */ + timeoutMs: number; + signal?: AbortSignal; +}): Promise { + const { invokeTool, apiKey, candidate, query, timeoutMs, signal } = input; + try { + const result = await invokeTool({ + serverName: SEARCH_WEB_SERVER_NAME, + serverUrl: candidate.endpoint, + toolName: SEARCH_WEB_TOOL_NAME, + arguments: { query, n: SEARCH_WEB_RESULT_COUNT }, + headers: { Authorization: `Bearer ${apiKey}` }, + timeoutMs, + ...(signal ? { signal } : {}), + }); + if (result.isError) { + const detail = result.content || `MCP tool ${SEARCH_WEB_SERVER_NAME}.${SEARCH_WEB_TOOL_NAME} failed`; + return { + ok: false, + failure: nameEndpoint(classifyFailure({ text: detail, isError: true }), candidate.endpoint), + }; + } + return { ok: true, outcome: readStepFunOutcome(result) }; + } catch (error) { + // The message carries no credential; it is surfaced verbatim so a + // profile/endpoint mismatch stays diagnosable, and cancellation is read + // from the signal rather than from the message. return { - content: [{ type: "text", text: content }], - details: { query, resultCount: 0, urls: [] }, + ok: false, + failure: nameEndpoint( + classifyFailure({ + text: error instanceof Error ? error.message : String(error), + cancelled: signal?.aborted === true, + }), + candidate.endpoint, + ), }; } +} + +/** + * A search failure names the endpoint that produced it: for the paid tier the + * endpoint decides which account is billed, so a profile/endpoint mismatch is + * otherwise invisible until it shows up on a bill. + */ +function nameEndpoint(failure: SearchFailure, endpoint: string): SearchFailure { + return { ...failure, detail: `${failure.detail} (endpoint ${endpoint})` }; +} - const results = items.map((item, index) => { +/** Read the StepFun payload without changing its existing rendering semantics. */ +function readStepFunOutcome(result: SearchWebMcpResult): SearchOutcome { + const items = result.structuredContent?.results; + if (!Array.isArray(items)) { + const text = result.content?.trim(); + return text ? { hits: [], text } : { hits: [] }; + } + const hits: SearchHit[] = []; + for (const item of items) { const record = isRecord(item) ? item : {}; const url = normalizeOptionalText(record.url); + if (!url) continue; const title = normalizeOptionalText(record.title); const snippet = shortenText( normalizeOptionalText(record.snippet) ?? normalizeOptionalText(record.content) ?? "", MAX_SNIPPET_CHARS, ); - return { - position: typeof record.position === "number" ? record.position : index + 1, - title, + hits.push({ url, - snippet, - }; - }); - - const urls = results.flatMap((result) => (result.url ? [result.url] : [])); - const content = results.length - ? results - .map((result) => { - const heading = result.url - ? `${result.position}. [${result.title ?? result.url}](${result.url})` - : `${result.position}. ${result.title ?? "(untitled)"}`; - return result.snippet ? `${heading}\n ${result.snippet}` : heading; - }) - .join("\n\n") - : `No search results for "${query}".`; + ...(title ? { title } : {}), + ...(snippet ? { snippet } : {}), + }); + if (hits.length >= SEARCH_WEB_RESULT_COUNT) break; + } + return { hits }; +} - return { - content: [{ type: "text", text: content }], - details: { query, resultCount: results.length, urls }, +function resolveSearchWebAuthPath(options: SearchWebToolOptions, env: Record): string { + return ( + normalizeOptionalText(options.authPath) ?? + normalizeOptionalText(env.STEPCODE_AUTH_PATH) ?? + join(resolveStepAgentDir(env), "auth.json") + ); +} + +function renderSearchResults(query: string, result: SearchChainResult): AgentToolResult { + const details: SearchWebDetails = { + query, + resultCount: result.outcome.hits.length, + urls: result.outcome.hits.flatMap((hit) => (hit.url ? [hit.url] : [])), + provider: result.provider, + degraded: result.degraded, }; + + if (result.outcome.hits.length === 0) { + const text = result.outcome.text?.trim() || `No search results for "${query}".`; + return { content: [{ type: "text", text }], details }; + } + + const content = result.outcome.hits + .map((hit, index) => { + const heading = `${index + 1}. [${hit.title ?? hit.url}](${hit.url})`; + return hit.snippet ? `${heading}\n ${shortenText(hit.snippet, MAX_SNIPPET_CHARS)}` : heading; + }) + .join("\n\n"); + + return { content: [{ type: "text", text: content }], details }; } function readCredentialToken(credential: Credential | undefined): string | undefined { diff --git a/packages/coding-agent/src/step/search/adapters.ts b/packages/coding-agent/src/step/search/adapters.ts new file mode 100644 index 000000000..da7c8cb8b --- /dev/null +++ b/packages/coding-agent/src/step/search/adapters.ts @@ -0,0 +1,286 @@ +import type { McpToolPayload, SearchHit, SearchOutcome, SearchRequest } from "./types.ts"; + +/** Result count requested from every provider and also the render cap. */ +export const SEARCH_RESULT_LIMIT = 10; + +export interface SearchAdapter { + readonly id: string; + readonly label: string; + readonly endpoint: string; + readonly toolName: string; + readonly transport: "mcp" | "http"; + /** Fixed request headers. None of these providers receive a credential. */ + readonly headers?: Record; + buildRequest(query: string): SearchRequest; + /** Map a successful payload to normalized hits. Throws only on malformed data. */ + parse(payload: unknown): SearchOutcome; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function text(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function positiveNumber(value: unknown): number | undefined { + const parsed = typeof value === "string" ? Date.parse(value) : typeof value === "number" ? value : Number.NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} + +function hit(record: Record, snippetKeys: readonly string[]): SearchHit | undefined { + const url = text(record.url); + if (!url) return undefined; + const snippet = snippetKeys.map((key) => text(record[key])).find((value) => value !== undefined); + const title = text(record.title); + return { + url, + ...(title ? { title } : {}), + ...(snippet ? { snippet } : {}), + }; +} + +function hitsFrom(records: unknown, snippetKeys: readonly string[]): SearchHit[] { + if (!Array.isArray(records)) return []; + const hits: SearchHit[] = []; + for (const record of records) { + if (!isRecord(record)) continue; + const normalized = hit(record, snippetKeys); + if (normalized) hits.push(normalized); + if (hits.length >= SEARCH_RESULT_LIMIT) break; + } + return hits; +} + +/** First text block of a stateless MCP payload. */ +function contentText(payload: unknown): string | undefined { + if (!isRecord(payload)) return undefined; + const blocks = payload.content; + if (!Array.isArray(blocks)) return undefined; + for (const block of blocks) { + if (isRecord(block) && block.type === "text") { + const value = text(block.text); + if (value) return value; + } + } + return undefined; +} + +function structured(payload: unknown): unknown { + return isRecord(payload) ? payload.structuredContent : undefined; +} + +function parseJson(value: string | undefined): unknown { + if (!value) return undefined; + try { + return JSON.parse(value) as unknown; + } catch { + return undefined; + } +} + +/** Firecrawl: `data.web[]` in structured content, falling back to the JSON text block. */ +const firecrawl: SearchAdapter = { + id: "firecrawl", + label: "Firecrawl", + endpoint: "https://mcp.firecrawl.dev/v2/mcp", + toolName: "firecrawl_search", + transport: "mcp", + buildRequest(query) { + return { + endpoint: this.endpoint, + toolName: this.toolName, + transport: "mcp", + args: { query, limit: SEARCH_RESULT_LIMIT }, + }; + }, + parse(payload) { + const fromStructured = readWeb(structured(payload)); + if (fromStructured.length) return { hits: fromStructured }; + const parsed = parseJson(contentText(payload)); + const hits = readWeb(parsed); + if (hits.length) return { hits }; + const raw = contentText(payload); + return raw ? { hits: [], text: raw } : { hits: [] }; + }, +}; + +function readWeb(value: unknown): SearchHit[] { + if (!isRecord(value)) return []; + const data = isRecord(value.data) ? value.data : undefined; + return hitsFrom(data?.web, ["description", "content", "snippet"]); +} + +/** + * Exa answers with one text block per result: + * + * Title: ... + * URL: https://... + * Published: N/A + * Highlights: + * ... + * + * Blocks are separated by a `---` rule. `Published: N/A` means the field is + * simply absent, not an error. + */ +const exa: SearchAdapter = { + id: "exa", + label: "Exa", + endpoint: "https://mcp.exa.ai/mcp", + toolName: "web_search_exa", + transport: "mcp", + buildRequest(query) { + return { + endpoint: this.endpoint, + toolName: this.toolName, + transport: "mcp", + args: { query, numResults: SEARCH_RESULT_LIMIT }, + }; + }, + parse(payload) { + const raw = contentText(payload); + if (!raw) return { hits: [] }; + const hits: SearchHit[] = []; + for (const block of raw.split(/\n\s*---\s*\n/u)) { + const url = block.match(/^URL:\s*(.+)$/mu)?.[1]?.trim(); + if (!url || url === "N/A") continue; + const title = block.match(/^Title:\s*(.+)$/mu)?.[1]?.trim(); + const published = block.match(/^Published:\s*(.+)$/mu)?.[1]?.trim(); + const body = block.match(/^(?:Highlights|Text|Summary):\s*\n?([\s\S]*)$/mu)?.[1]?.trim(); + hits.push({ + url, + ...(title && title !== "N/A" ? { title } : {}), + ...(body ? { snippet: body } : {}), + ...(published && published !== "N/A" ? publishedAt(published) : {}), + }); + if (hits.length >= SEARCH_RESULT_LIMIT) break; + } + return hits.length ? { hits } : { hits: [], text: raw }; + }, +}; + +function publishedAt(value: string): { publishedAtMs?: number } { + const parsed = positiveNumber(value); + return parsed === undefined ? {} : { publishedAtMs: parsed }; +} + +/** Parallel: `results[]` in structured content, with excerpts as the snippet. */ +const parallel: SearchAdapter = { + id: "parallel", + label: "Parallel", + endpoint: "https://search.parallel.ai/mcp", + toolName: "web_search", + transport: "mcp", + buildRequest(query) { + return { + endpoint: this.endpoint, + toolName: this.toolName, + transport: "mcp", + args: { objective: query, search_queries: [query] }, + }; + }, + parse(payload) { + const fromStructured = readParallel(structured(payload)); + if (fromStructured.length) return { hits: fromStructured }; + const parsed = parseJson(contentText(payload)); + const hits = readParallel(parsed); + if (hits.length) return { hits }; + const raw = contentText(payload); + return raw ? { hits: [], text: raw } : { hits: [] }; + }, +}; + +function readParallel(value: unknown): SearchHit[] { + if (!isRecord(value) || !Array.isArray(value.results)) return []; + const hits: SearchHit[] = []; + for (const entry of value.results) { + if (!isRecord(entry)) continue; + const url = text(entry.url); + if (!url) continue; + const title = text(entry.title); + const excerpts = Array.isArray(entry.excerpts) + ? entry.excerpts.filter((item): item is string => typeof item === "string" && item.trim().length > 0) + : []; + hits.push({ + url, + ...(title ? { title } : {}), + ...(excerpts.length ? { snippet: excerpts.join("\n\n") } : {}), + ...(entry.publish_date ? publishedAt(String(entry.publish_date)) : {}), + }); + if (hits.length >= SEARCH_RESULT_LIMIT) break; + } + return hits; +} + +/** TinyFish: JSON inside the text block; the keyless mode needs only the access-mode header. */ +const tinyfish: SearchAdapter = { + id: "tinyfish", + label: "TinyFish", + endpoint: "https://agent.tinyfish.ai/mcp", + toolName: "search", + transport: "mcp", + headers: { "X-TinyFish-Access-Mode": "keyless" }, + buildRequest(query) { + return { + endpoint: this.endpoint, + toolName: this.toolName, + transport: "mcp", + args: { query }, + headers: this.headers, + }; + }, + parse(payload) { + const parsed = parseJson(contentText(payload)); + if (isRecord(parsed)) { + const hits = hitsFrom(parsed.results, ["snippet", "content", "description"]); + if (hits.length) return { hits }; + } + const raw = contentText(payload); + return raw ? { hits: [], text: raw } : { hits: [] }; + }, +}; + +/** Tavily is a plain JSON endpoint, not MCP. */ +const tavily: SearchAdapter = { + id: "tavily", + label: "Tavily", + endpoint: "https://api.tavily.com/search", + toolName: "search", + transport: "http", + headers: { "X-Tavily-Access-Mode": "keyless" }, + buildRequest(query) { + return { + endpoint: this.endpoint, + toolName: this.toolName, + transport: "http", + args: { + query, + search_depth: "basic", + chunks_per_source: 3, + max_results: SEARCH_RESULT_LIMIT, + }, + headers: this.headers, + }; + }, + parse(payload) { + if (!isRecord(payload)) return { hits: [] }; + return { hits: hitsFrom(payload.results, ["content", "snippet", "description"]) }; + }, +}; + +/** + * Free, credential-less providers. The order here is only the declaration order; + * `buildSearchChain` shuffles it per call so traffic is spread across providers + * instead of always consuming the first one's daily allowance. + */ +export const KEYLESS_ADAPTERS: readonly SearchAdapter[] = [exa, firecrawl, tavily, parallel, tinyfish]; + +export const ADAPTERS: readonly SearchAdapter[] = KEYLESS_ADAPTERS; + +/** Look up an adapter by the provider id used in configuration. */ +export function findAdapter(id: string): SearchAdapter | undefined { + return ADAPTERS.find((adapter) => adapter.id === id); +} + +export type { McpToolPayload }; diff --git a/packages/coding-agent/src/step/search/errors.ts b/packages/coding-agent/src/step/search/errors.ts new file mode 100644 index 000000000..12d246858 --- /dev/null +++ b/packages/coding-agent/src/step/search/errors.ts @@ -0,0 +1,111 @@ +import type { SearchFailure, SearchFailureKind } from "./types.ts"; + +/** + * Everything a transport can observe about one failed call. Providers disagree + * on how they report failure, so classification never relies on a single field: + * - some answer HTTP 200 with `isError: true` and the reason in the text, + * - some answer a non-2xx status with a JSON body, + * - some answer a JSON-RPC error object. + * + * Treating only the HTTP status as the signal would report those `isError` + * answers as "no results", which also stops the fallback chain from advancing. + */ +export interface FailureInput { + status?: number; + /** JSON-RPC error code and message, when the body carried one. */ + jsonrpcError?: { code: number; message: string }; + /** MCP tool result flag. */ + isError?: boolean; + /** Response text, used for both `isError` reasons and status-code fallbacks. */ + text?: string; + /** Transport-level error (DNS, TLS, reset) rather than a response. */ + networkError?: string; + /** The caller's own signal fired; this is a user cancellation, not a timeout. */ + cancelled?: boolean; + /** `Retry-After` in milliseconds, when a response carried one. */ + retryAfterMs?: number; +} + +/** Patterns are matched against the provider's own text; order matters. */ +const AUTH_PATTERN = /unauthori[sz]ed|forbidden|invalid[^.]{0,20}key|api key is invalid|authentication failed/iu; +const RATE_LIMIT_PATTERN = /rate limit|too many requests|throttl/iu; +// Providers spell the daily ceiling several ways: `daily_cap_reached`, +// `daily limit`, `reached the daily ... limit`. +const QUOTA_PATTERN = + /quota|credits?\b|daily[_\s-]?cap|daily[_\s-]?limit|usage limit|out of credits|reached (?:the|your) daily/iu; +const SERVER_PATTERN = /internal server error|bad gateway|service unavailable|gateway timeout/iu; + +function kindFromStatus(status: number): SearchFailureKind | undefined { + if (status === 401 || status === 403) return "auth"; + if (status === 402) return "quota"; + if (status === 429) return "rate_limit"; + if (status >= 500) return "server"; + return undefined; +} + +function kindFromText(text: string): SearchFailureKind | undefined { + if (AUTH_PATTERN.test(text)) return "auth"; + if (QUOTA_PATTERN.test(text)) return "quota"; + if (RATE_LIMIT_PATTERN.test(text)) return "rate_limit"; + if (SERVER_PATTERN.test(text)) return "server"; + return undefined; +} + +/** + * Classify one failure. Precedence is `isError` text, then HTTP status, then + * status-shaped text, then unknown. + * + * The text first order matters: a provider that answers 200 with an `isError` + * body reports the same conditions (auth, quota, rate limit) that others report + * through a status code, and the chain must treat both identically. + */ +export function classifyFailure(input: FailureInput): SearchFailure { + if (input.cancelled) { + return { kind: "cancelled", detail: "search cancelled" }; + } + if (input.networkError) { + return { kind: "network", detail: input.networkError, ...retryAfter(input.retryAfterMs) }; + } + + const text = input.text?.trim(); + const fromText = text ? kindFromText(text) : undefined; + const fromStatus = input.status === undefined ? undefined : kindFromStatus(input.status); + // An `isError` body is the provider's own verdict, so its text outranks the + // status code it was delivered with. + let kind: SearchFailureKind = input.isError + ? (fromText ?? fromStatus ?? "unknown") + : (fromStatus ?? fromText ?? "unknown"); + // A daily ceiling arrives under the generic 429 code. It is a quota, not a + // burst limit: naming it correctly matters for diagnostics, and the same + // call would otherwise look like a transient spike. + if (kind === "rate_limit" && fromText === "quota") kind = "quota"; + + if (input.jsonrpcError && kind === "unknown") { + return { + kind: "unsupported", + detail: `JSON-RPC ${input.jsonrpcError.code}: ${input.jsonrpcError.message}`, + ...(input.status === undefined ? {} : { status: input.status }), + }; + } + + return { + kind, + detail: text || `HTTP ${input.status ?? "?"}`, + ...(input.status === undefined ? {} : { status: input.status }), + ...retryAfter(input.retryAfterMs), + }; +} + +function retryAfter(retryAfterMs: number | undefined): { retryAfterMs?: number } { + return retryAfterMs === undefined ? {} : { retryAfterMs }; +} + +/** Parse a `Retry-After` header as milliseconds; both the seconds and date forms are allowed. */ +export function parseRetryAfter(value: string | null | undefined, now = Date.now()): number | undefined { + const trimmed = value?.trim(); + if (!trimmed) return undefined; + const seconds = Number(trimmed); + if (Number.isFinite(seconds)) return seconds >= 0 ? seconds * 1000 : undefined; + const at = Date.parse(trimmed); + return Number.isFinite(at) ? Math.max(0, at - now) : undefined; +} diff --git a/packages/coding-agent/src/step/search/routing.ts b/packages/coding-agent/src/step/search/routing.ts new file mode 100644 index 000000000..50c8f9901 --- /dev/null +++ b/packages/coding-agent/src/step/search/routing.ts @@ -0,0 +1,92 @@ +import { KEYLESS_ADAPTERS } from "./adapters.ts"; +import type { SearchCandidate, SearchChain, SearchChainMode } from "./types.ts"; + +export interface BuildSearchChainInput { + readonly mode: SearchChainMode; + /** Whether the Step login resolves a search credential for this call. */ + readonly hasStepCredential: boolean; + /** Resolved StepFun search endpoint. The caller always resolves one. */ + readonly stepEndpoint: string; + /** Injected for deterministic tests; defaults to `Math.random`. */ + readonly random?: () => number; +} + +/** The StepFun search tool name; the endpoint differs per login profile. */ +export const STEPFUN_SEARCH_TOOL_NAME = "web_search"; + +function fisherYates(items: readonly T[], random: () => number): T[] { + const shuffled = [...items]; + for (let index = shuffled.length - 1; index > 0; index -= 1) { + const swap = Math.floor(random() * (index + 1)); + const value = shuffled[index]!; + shuffled[index] = shuffled[swap]!; + shuffled[swap] = value; + } + return shuffled; +} + +function keylessCandidates(random: () => number): SearchCandidate[] { + return fisherYates(KEYLESS_ADAPTERS, random).map((adapter) => ({ + id: adapter.id, + tier: "keyless", + endpoint: adapter.endpoint, + toolName: adapter.toolName, + transport: adapter.transport, + // Free endpoints are called statelessly; a refusal falls back to a handshake. + allowHandshake: true, + ...(adapter.headers ? { headers: adapter.headers } : {}), + })); +} + +function stepfunCandidate(endpoint: string): SearchCandidate { + return { + id: "stepfun", + tier: "stepfun", + endpoint, + toolName: STEPFUN_SEARCH_TOOL_NAME, + transport: "mcp", + // The StepFun endpoint is reached through the SDK client already. + allowHandshake: false, + }; +} + +/** + * Build the ordered candidate list for one search. + * + * The paid tier is only present when a Step search credential actually resolves + * for this call. An absent credential is not "try and fail": the free providers + * are used directly, which skips a call that cannot succeed. + * + * Free providers are shuffled per call rather than fixed: every provider has its + * own free allowance, so a fixed order would spend the first one's allowance on + * all traffic and leave the rest idle. Shuffling spreads the load while the + * chain still walks the list in order. + */ +export function buildSearchChain(input: BuildSearchChainInput): SearchChain { + const random = input.random ?? Math.random; + + if (input.mode === "off") { + return { candidates: [], disabled: true, skippedPaidTier: "disabled" }; + } + + if (input.mode === "step") { + // Pinned to the paid backend. `stepEndpoint` is always resolved by the + // caller, so the only thing that can be missing is the credential, which is + // a configuration error rather than a reason to use another backend. + if (!input.hasStepCredential) return { candidates: [], requiresCredential: true }; + return { candidates: [stepfunCandidate(input.stepEndpoint)] }; + } + + if (input.mode !== "auto") { + const adapter = KEYLESS_ADAPTERS.find((entry) => entry.id === input.mode); + if (!adapter) return { candidates: [], disabled: true }; + const [candidate] = keylessCandidates(random).filter((entry) => entry.id === adapter.id); + return { candidates: candidate ? [candidate] : [] }; + } + + const free = keylessCandidates(random); + if (input.hasStepCredential) { + return { candidates: [stepfunCandidate(input.stepEndpoint), ...free] }; + } + return { candidates: free, skippedPaidTier: "no-step-credential" }; +} diff --git a/packages/coding-agent/src/step/search/scheduler.ts b/packages/coding-agent/src/step/search/scheduler.ts new file mode 100644 index 000000000..813cbcb0b --- /dev/null +++ b/packages/coding-agent/src/step/search/scheduler.ts @@ -0,0 +1,202 @@ +import { SEARCH_ATTEMPT_TIMEOUT_MS, SEARCH_TOTAL_BUDGET_MS } from "./transport.ts"; +import type { SearchAttempt, SearchCandidate, SearchChain, SearchOutcome, SearchTier } from "./types.ts"; +import { advancesChain } from "./types.ts"; + +/** Skip an unavailable candidate long enough for a transient allowance or outage to pass. */ +export const SEARCH_COOLDOWN_MS = 60_000; + +/** A rejected credential is not transient; stop retrying it for longer. */ +export const SEARCH_AUTH_COOLDOWN_MS = 5 * 60_000; + +/** + * A free provider that rejects an unauthenticated request is not having a bad + * five minutes: its keyless mode is gone. Retrying it every few minutes wastes a + * round trip forever, so it stays out for the life of the process. Only a restart + * (or removing the provider) changes the outcome. + */ +const disabledKeyless = new Map(); + +/** Below this remaining budget one more attempt could not finish a fast failure. */ +export const MIN_ATTEMPT_MS = 3_000; + +/** + * Cooldowns live at module scope rather than per tool instance: the tool is + * rebuilt for every session, so a per-instance map would forget which endpoint + * is exhausted and every new session would pay the failed call again. + */ +const cooldowns = new Map(); + +/** Cooldown identity: a candidate is (provider, endpoint), not just provider id. */ +function cooldownKey(candidate: SearchCandidate): string { + return `${candidate.id}\u0000${candidate.endpoint}`; +} + +/** Clear recorded cooldowns. Test-only. */ +export function resetSearchCooldowns(): void { + cooldowns.clear(); + disabledKeyless.clear(); +} + +function isCooling(key: string, now: number): boolean { + const until = cooldowns.get(key); + if (until === undefined) return false; + if (until <= now) { + cooldowns.delete(key); + return false; + } + return true; +} + +export interface SearchAttemptRecord { + candidate: string; + kind: string; + detail: string; + endpoint: string; +} + +export interface SearchChainResult { + outcome: SearchOutcome; + /** Candidate that produced the answer. */ + provider: string; + tier: SearchTier; + /** True when the answer did not come from the paid tier. */ + degraded: boolean; + attempts: SearchAttemptRecord[]; + /** Candidates skipped because they were cooling down. */ + skipped: string[]; +} + +export type SearchCandidateInvoker = ( + candidate: SearchCandidate, + query: string, + options: { signal?: AbortSignal; timeoutMs: number }, +) => Promise; + +export interface RunSearchChainInput { + chain: SearchChain; + query: string; + invoke: SearchCandidateInvoker; + signal?: AbortSignal; + now?: () => number; + budgetMs?: number; +} + +/** + * Walk the candidate list until one answers. + * + * Each attempt gets a slice of one shared budget, so a slow candidate cannot + * starve the rest of the chain. A user cancellation ends the walk immediately: + * no further provider may be contacted after the caller asked to stop. + */ +export async function runSearchChain(input: RunSearchChainInput): Promise { + if (input.chain.disabled) + throw new Error("search_web is disabled; set [search] provider in config.toml to enable it"); + if (input.chain.candidates.length === 0) { + throw new Error("search_web has no available provider; run `/login` or configure [search] provider"); + } + + const now = input.now ?? Date.now; + const budget = input.budgetMs ?? SEARCH_TOTAL_BUDGET_MS; + const started = now(); + const deadline = started + budget; + const attempts: SearchAttemptRecord[] = []; + const skipped: string[] = []; + + for (const candidate of input.chain.candidates) { + if (input.signal?.aborted) break; + const at = now(); + const key = cooldownKey(candidate); + if (disabledKeyless.has(key)) { + skipped.push(candidate.id); + continue; + } + if (isCooling(key, at)) { + skipped.push(candidate.id); + continue; + } + const remaining = deadline - at; + if (remaining < MIN_ATTEMPT_MS) { + attempts.push({ + candidate: candidate.id, + kind: "budget", + detail: "skipped: search budget exhausted", + endpoint: candidate.endpoint, + }); + break; + } + + const attempt = await input.invoke(candidate, input.query, { + signal: input.signal, + timeoutMs: Math.min(SEARCH_ATTEMPT_TIMEOUT_MS, remaining), + }); + + if (attempt.ok) { + return { + outcome: attempt.outcome, + provider: candidate.id, + tier: candidate.tier, + degraded: candidate.tier !== "stepfun", + attempts, + skipped, + }; + } + + attempts.push({ + candidate: candidate.id, + kind: attempt.failure.kind, + detail: attempt.failure.detail, + endpoint: candidate.endpoint, + }); + + if (!advancesChain(attempt.failure.kind)) break; + if (attempt.failure.kind === "auth" && candidate.tier === "keyless") { + // A free endpoint that rejects an unauthenticated call has lost its + // keyless mode; that is permanent for this process, not a transient fault. + disabledKeyless.set(key, attempt.failure.detail); + continue; + } + cooldowns.set( + key, + now() + + (attempt.failure.kind === "auth" + ? SEARCH_AUTH_COOLDOWN_MS + : (attempt.failure.retryAfterMs ?? SEARCH_COOLDOWN_MS)), + ); + } + + if (input.signal?.aborted) throw new Error("search_web cancelled"); + if (attempts.length === 0 && skipped.length > 0) { + // Every candidate is deliberately skipped. Say so instead of reporting a + // failure with no attempts, which reads like a bug. A permanently disabled + // provider is named separately: waiting will not bring it back. + const disabled = skipped.filter((id) => { + const candidate = input.chain.candidates.find((entry) => entry.id === id); + return candidate ? disabledKeyless.has(cooldownKey(candidate)) : false; + }); + const cooling = skipped.filter((id) => !disabled.includes(id)); + const parts: string[] = []; + if (cooling.length) { + const soonest = Math.min( + ...cooling.map((id) => { + const candidate = input.chain.candidates.find((entry) => entry.id === id); + return candidate ? (cooldowns.get(cooldownKey(candidate)) ?? now()) : now(); + }), + ); + const seconds = Math.max(1, Math.ceil((soonest - now()) / 1000)); + parts.push(`${cooling.join(", ")} cooling down, retry in about ${seconds}s`); + } + if (disabled.length) { + parts.push(`${disabled.join(", ")} no longer accepts credential-free requests`); + } + throw new Error(`search_web has no usable provider: ${parts.join("; ")}`); + } + throw new Error(formatFailure(attempts, skipped)); +} + +function formatFailure(attempts: SearchAttemptRecord[], skipped: string[]): string { + const lines = attempts.map( + (attempt) => `${attempt.candidate}: ${attempt.kind} (${attempt.detail}) [endpoint ${attempt.endpoint}]`, + ); + if (skipped.length) lines.push(`skipped while cooling down: ${skipped.join(", ")}`); + return `search_web failed after ${attempts.length} attempt(s): ${lines.join("; ")}`; +} diff --git a/packages/coding-agent/src/step/search/transport.ts b/packages/coding-agent/src/step/search/transport.ts new file mode 100644 index 000000000..f91e00fbe --- /dev/null +++ b/packages/coding-agent/src/step/search/transport.ts @@ -0,0 +1,288 @@ +import { invokeRemoteMcpTool, type RemoteMcpToolResult } from "../mcp-client.ts"; +import { STEPCODE_VERSION } from "../version.ts"; +import { classifyFailure, parseRetryAfter } from "./errors.ts"; +import type { McpToolPayload, SearchAttempt, SearchFailure, SearchOutcome, SearchRequest } from "./types.ts"; + +/** Response bodies above this size are rejected instead of buffered in full. */ +export const MAX_SEARCH_RESPONSE_BYTES = 1024 * 1024; + +/** Time allowed for one candidate before the chain moves on. */ +export const SEARCH_ATTEMPT_TIMEOUT_MS = 8_000; + +/** Time allowed for the whole chain, including every fallback. */ +export const SEARCH_TOTAL_BUDGET_MS = 25_000; + +/** + * Provider endpoints apply their own bot filtering, and at least one rejects + * generic HTTP-library user agents outright. Identify the product on every call. + */ +function userAgent(): string { + return `StepCode/${STEPCODE_VERSION.value}`; +} + +function requestHeaders(request: SearchRequest): Record { + return { + "User-Agent": userAgent(), + Accept: "application/json, text/event-stream", + "Content-Type": "application/json", + ...request.headers, + }; +} + +type Fetched = { payload: unknown } | { failure: SearchFailure }; + +interface JsonRpcEnvelope { + result?: McpToolPayload; + error?: { code: number; message: string }; +} + +function parseJson(raw: string): unknown { + const trimmed = raw.trim(); + if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return undefined; + try { + return JSON.parse(trimmed) as unknown; + } catch { + return undefined; + } +} + +function asEnvelope(value: unknown): JsonRpcEnvelope | undefined { + if (typeof value !== "object" || value === null) return undefined; + const record = value as Record; + if (typeof record.error === "object" && record.error !== null) { + const detail = record.error as Record; + return { + error: { + code: typeof detail.code === "number" ? detail.code : -1, + message: typeof detail.message === "string" ? detail.message : "unknown JSON-RPC error", + }, + }; + } + if (typeof record.result === "object" && record.result !== null) { + return { result: record.result as McpToolPayload }; + } + return undefined; +} + +/** + * Read a JSON-RPC envelope from either a plain JSON body or an SSE stream. + * The stream form is what MCP servers negotiate by default, so both are needed. + */ +function readEnvelope(body: string, contentType: string | null): JsonRpcEnvelope | undefined { + const direct = asEnvelope(parseJson(body)); + if (direct) return direct; + if (contentType?.includes("text/event-stream") || body.includes("data: ")) { + for (const line of body.split("\n")) { + if (!line.startsWith("data: ")) continue; + const parsed = asEnvelope(parseJson(line.slice(6))); + if (parsed) return parsed; + } + } + return undefined; +} + +/** + * Build the request body. MCP candidates need a JSON-RPC `tools/call` envelope; + * sending the bare arguments is rejected as a parse error. Tavily is a plain + * JSON endpoint and takes the arguments directly. + */ +function requestBody(request: SearchRequest): string { + if (request.transport === "http") return JSON.stringify(request.args); + return JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { name: request.toolName, arguments: request.args }, + }); +} + +async function fetchOnce( + request: SearchRequest, + timeoutMs: number, + callerSignal: AbortSignal | undefined, +): Promise { + const timeoutSignal = AbortSignal.timeout(timeoutMs); + const signal = callerSignal ? AbortSignal.any([callerSignal, timeoutSignal]) : timeoutSignal; + const cancelled = () => callerSignal?.aborted === true; + + let response: Response; + try { + response = await fetch(request.endpoint, { + method: "POST", + headers: requestHeaders(request), + body: requestBody(request), + signal, + }); + } catch (error) { + return { + failure: classifyFailure({ + networkError: error instanceof Error ? error.message : String(error), + cancelled: cancelled(), + }), + }; + } + + const retryAfterMs = parseRetryAfter(response.headers.get("retry-after")); + let body: string; + try { + body = await response.text(); + } catch (error) { + return { + failure: classifyFailure({ + status: response.status, + networkError: error instanceof Error ? error.message : String(error), + cancelled: cancelled(), + }), + }; + } + if (Buffer.byteLength(body, "utf8") > MAX_SEARCH_RESPONSE_BYTES) { + return { + failure: { + kind: "parse", + status: response.status, + detail: `response exceeded ${MAX_SEARCH_RESPONSE_BYTES} bytes`, + }, + }; + } + + if (request.transport === "http") { + if (!response.ok) { + return { + failure: classifyFailure({ status: response.status, text: body, retryAfterMs }), + }; + } + const payload = parseJson(body); + return payload === undefined + ? { failure: { kind: "parse", status: response.status, detail: body.slice(0, 500) || "empty response" } } + : { payload }; + } + + const envelope = readEnvelope(body, response.headers.get("content-type")); + if (!envelope) { + return { + failure: classifyFailure({ + status: response.status, + text: body.slice(0, 500) || `HTTP ${response.status} without a JSON-RPC response`, + retryAfterMs, + }), + }; + } + if (envelope.error) { + const failure = classifyFailure({ + status: response.status, + jsonrpcError: envelope.error, + text: envelope.error.message, + retryAfterMs, + }); + // Some servers refuse a stateless call and only answer after `initialize`. + // That is a protocol mismatch, not a provider outage, so the caller may + // retry the same candidate once with a handshake. An auth or quota failure + // carrying the same envelope is a real answer: retrying it would only + // double the latency. + const retryable = failure.kind === "unsupported" || failure.kind === "unknown"; + return { failure: retryable ? { ...failure, handshakeFallback: true } : failure }; + } + const payload = envelope.result ?? {}; + if (payload.isError === true) { + const detail = payload.content?.find((block) => block.type === "text")?.text ?? "search request failed"; + // The provider's own verdict outranks the status it arrived with: several + // answer 200 and report a quota or auth problem in the body. The status and + // Retry-After still travel with the failure so a rate limiter that reports + // through the body keeps its own backoff. + return { + failure: classifyFailure({ text: detail, isError: true, status: response.status, retryAfterMs }), + }; + } + return { payload }; +} + +/** + * One search against one candidate. + * + * Free providers are called with a single stateless `tools/call`, skipping the + * MCP handshake and catalog listing that the StepFun endpoint performs: those + * two round trips were the bulk of a short search's latency. A provider that + * refuses the stateless form is retried once through the SDK client. + */ +/** + * The SDK client reports `content` as one joined string, while the adapters read + * the MCP block list. Convert instead of handing over a shape they cannot read: + * a string there would parse as "no results" and end the chain on a false empty + * answer. + */ +function fromRemoteMcpResult(result: RemoteMcpToolResult): McpToolPayload { + return { + ...(result.isError === true ? { isError: true } : {}), + ...(result.content ? { content: [{ type: "text", text: result.content }] } : {}), + ...(result.structuredContent ? { structuredContent: result.structuredContent } : {}), + }; +} + +export async function callSearchProvider(input: { + request: SearchRequest; + parse: (payload: unknown) => SearchOutcome; + timeoutMs: number; + signal?: AbortSignal; + allowHandshake: boolean; + serverName: string; +}): Promise { + const { request, signal, timeoutMs } = input; + signal?.throwIfAborted(); + + const fetched = await fetchOnce(request, timeoutMs, signal); + if ("payload" in fetched) { + try { + return { ok: true, outcome: input.parse(fetched.payload) }; + } catch (error) { + return { + ok: false, + failure: { + kind: "parse", + detail: error instanceof Error ? error.message : String(error), + }, + }; + } + } + + if (!input.allowHandshake || fetched.failure.handshakeFallback !== true || signal?.aborted) { + return { ok: false, failure: fetched.failure }; + } + + try { + const result = await invokeRemoteMcpTool({ + serverName: input.serverName, + serverUrl: request.endpoint, + toolName: request.toolName, + arguments: request.args, + headers: request.headers, + timeoutMs, + signal, + }); + if (result.isError) { + return { + ok: false, + failure: classifyFailure({ + text: result.content ?? "search request failed", + isError: true, + cancelled: signal?.aborted === true, + }), + }; + } + try { + return { ok: true, outcome: input.parse(fromRemoteMcpResult(result)) }; + } catch (error) { + return { + ok: false, + failure: { kind: "parse", detail: error instanceof Error ? error.message : String(error) }, + }; + } + } catch (error) { + return { + ok: false, + failure: classifyFailure({ + networkError: error instanceof Error ? error.message : String(error), + cancelled: signal?.aborted === true, + }), + }; + } +} diff --git a/packages/coding-agent/src/step/search/types.ts b/packages/coding-agent/src/step/search/types.ts new file mode 100644 index 000000000..749e83079 --- /dev/null +++ b/packages/coding-agent/src/step/search/types.ts @@ -0,0 +1,117 @@ +/** One normalized search result, independent of which provider produced it. */ +export interface SearchHit { + url: string; + title?: string; + snippet?: string; + /** Publication time in milliseconds since the Unix epoch, when the provider reports one. */ + publishedAtMs?: number; +} + +/** + * A successful search. `text` keeps a provider's plain-text answer for the case + * where it replies without a result list, so the tool can pass it through + * instead of reporting "no results". + */ +export interface SearchOutcome { + hits: SearchHit[]; + text?: string; +} + +/** + * Why one candidate failed. The kind drives two separate decisions: whether the + * chain advances (`advanceable`) and how long the candidate is skipped for. + */ +export type SearchFailureKind = + | "auth" + | "rate_limit" + | "quota" + | "server" + | "network" + | "parse" + | "cancelled" + | "unsupported" + | "unknown"; + +export interface SearchFailure { + kind: SearchFailureKind; + /** HTTP status when a response was received. */ + status?: number; + /** Provider-facing reason. Never contains a credential. */ + detail: string; + /** Server-provided backoff for rate limiting, in milliseconds. */ + retryAfterMs?: number; + /** + * The endpoint did not accept a stateless `tools/call`. The caller may retry + * the same candidate once through a full MCP handshake. + */ + handshakeFallback?: boolean; +} + +export type SearchAttempt = { ok: true; outcome: SearchOutcome } | { ok: false; failure: SearchFailure }; + +/** + * Kinds that keep the chain moving. `cancelled` never advances: the user asked + * to stop, so no further request may be made. `auth` does advance, because a + * StepFun-only failure must still fall back to the free tier. + */ +export function advancesChain(kind: SearchFailureKind): boolean { + return kind !== "cancelled"; +} + +/** Attribute of a candidate used to decide whether the answer came from the paid tier. */ +export type SearchTier = "stepfun" | "keyless"; + +export interface SearchCandidate { + /** Stable identity used for cooldowns and diagnostics. */ + id: string; + tier: SearchTier; + endpoint: string; + toolName: string; + transport: "mcp" | "http"; + /** Stateless MCP candidates may need one full handshake before a real call. */ + allowHandshake: boolean; + headers?: Record; +} + +export interface SearchChain { + /** Candidates in the order they will be attempted. Empty means search is disabled. */ + candidates: SearchCandidate[]; + /** Why the paid tier is absent, for diagnostics. */ + skippedPaidTier?: "no-step-credential" | "disabled"; + disabled?: boolean; + /** + * The user pinned `step` but no credential resolves. This is a real + * configuration error: falling back to a free provider would silently use a + * different backend than the one that was asked for. + */ + requiresCredential?: boolean; +} + +export type SearchChainMode = "auto" | "off" | "step" | "exa" | "firecrawl" | "parallel" | "tavily" | "tinyfish"; + +export const SEARCH_CHAIN_MODES: readonly SearchChainMode[] = [ + "auto", + "off", + "step", + "exa", + "firecrawl", + "parallel", + "tavily", + "tinyfish", +]; + +/** A stateless `tools/call` response, reduced to the parts every adapter reads. */ +export interface McpToolPayload { + content?: { type?: string; text?: string }[]; + structuredContent?: unknown; + isError?: boolean; +} + +/** One provider call, fully resolved by an adapter. */ +export interface SearchRequest { + endpoint: string; + toolName: string; + transport: "mcp" | "http"; + args: Record; + headers?: Record; +} diff --git a/packages/coding-agent/test/fixtures/search/exa.json b/packages/coding-agent/test/fixtures/search/exa.json new file mode 100644 index 000000000..3fd4bba2f --- /dev/null +++ b/packages/coding-agent/test/fixtures/search/exa.json @@ -0,0 +1,15 @@ +{ + "result": { + "content": [ + { + "type": "text", + "text": "Title: The Async Ecosystem - Asynchronous Programming in Rust\nURL: https://rust-lang.github.io/async-book/08_ecosystem/00_chapter.html\nPublished: N/A\nAuthor: N/A\nHighlights:\nRust currently provides only the bare essentials for writing async code. Importantly, executors, tasks, reactors, combinators, and low-level I/O futures and traits are not yet provided in the standard library. In the meantime,…", + "_meta": { + "searchTime": 132.3 + } + } + ] + }, + "jsonrpc": "2.0", + "id": 1 +} \ No newline at end of file diff --git a/packages/coding-agent/test/fixtures/search/firecrawl.json b/packages/coding-agent/test/fixtures/search/firecrawl.json new file mode 100644 index 000000000..380142009 --- /dev/null +++ b/packages/coding-agent/test/fixtures/search/firecrawl.json @@ -0,0 +1,44 @@ +{ + "result": { + "content": [ + { + "type": "text", + "text": "{\"success\": true, \"data\": {\"web\": [{\"url\": \"https://corrode.dev/blog/async/\", \"title\": \"The State of Async Rust: Runtimes\", \"description\": \"## [Why Bother?](https://corrode.dev/blog/async/#why-bother)\\nIn the first article, we will focus on the current state of async Rust runtimes,\\n\\n## [Other Runtimes](https://corrode.d…\", \"position\": 1, \"source\": \"web\"}, {\"url\": \"https://rust-lang.github.io/async-book/08_ecosystem/00_chapter.html\", \"title\": \"The Async Ecosystem - Asynchronous Programming in Rust\", \"description\": \"# Asynchronous Programming in Rust\\nAsync runtimes are libraries used for executing async applications. Runtimes usually bundle together a _reactor_ with one or more _executors_. In…\", \"position\": 2, \"source\": \"web\"}, {\"url\": \"https://users.rust-lang.org/t/can-someone-explain-how-runtimes-for-async-and-await-work-on-an-assembly-level/130413\", \"title\": \"Can someone explain how runtimes for async and await work on an ...\", \"description\": \"Rust's async is a stackless coroutine. when one async function calls another Rust combines them into a bigger coroutine, thus creating an ...\", \"position\": 3, \"source\": \"web\"}]}, \"creditsUsed\": 2, \"agent_hints\": [\"All 3 web results are excerpts only (#1 \\\"https://corrode.dev/blog/async/\\\", #2 \\\"https://rust-lang.github.io/async-book/08_ecosystem/00_chapter.html\\\", #3 \\\"https://users.rust-lang.org…\"]}" + } + ], + "structuredContent": { + "success": true, + "data": { + "web": [ + { + "url": "https://corrode.dev/blog/async/", + "title": "The State of Async Rust: Runtimes", + "description": "## [Why Bother?](https://corrode.dev/blog/async/#why-bother)\nIn the first article, we will focus on the current state of async Rust runtimes,\n\n## [Other Runtimes](https://corrode.d…", + "position": 1, + "source": "web" + }, + { + "url": "https://rust-lang.github.io/async-book/08_ecosystem/00_chapter.html", + "title": "The Async Ecosystem - Asynchronous Programming in Rust", + "description": "# Asynchronous Programming in Rust\nAsync runtimes are libraries used for executing async applications. Runtimes usually bundle together a _reactor_ with one or more _executors_. In…", + "position": 2, + "source": "web" + }, + { + "url": "https://users.rust-lang.org/t/can-someone-explain-how-runtimes-for-async-and-await-work-on-an-assembly-level/130413", + "title": "Can someone explain how runtimes for async and await work on an ...", + "description": "Rust's async is a stackless coroutine. when one async function calls another Rust combines them into a bigger coroutine, thus creating an ...", + "position": 3, + "source": "web" + } + ] + }, + "creditsUsed": 2, + "agent_hints": [ + "All 3 web results are excerpts only (#1 \"https://corrode.dev/blog/async/\", #2 \"https://rust-lang.github.io/async-book/08_ecosystem/00_chapter.html\", #3 \"https://users.rust-lang.org…" + ] + } + }, + "jsonrpc": "2.0", + "id": 1 +} \ No newline at end of file diff --git a/packages/coding-agent/test/fixtures/search/parallel.json b/packages/coding-agent/test/fixtures/search/parallel.json new file mode 100644 index 000000000..d74f7d631 --- /dev/null +++ b/packages/coding-agent/test/fixtures/search/parallel.json @@ -0,0 +1,45 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "content": [ + { + "text": "{\"search_id\": \"search_9bbd1740a684d0ea2cefa7a63dc5355c\", \"results\": [{\"url\": \"https://rust-lang.github.io/async-book/08_ecosystem/00_chapter.html\", \"title\": \"The Async Ecosystem - Asynchronous Programming in Rust\", \"publish_date\": null, \"excerpts\": [\"* Auto\\n* Light\\n* Rust\\n* Coal\\n* Navy\\n* Ayu\\n\\nAsynchronous Programming in Rust\\n\\n \\n\\nThe Async Ecosystem\\n\\nRust currently provides only the bare essentials for writing async code. Import…\"]}, {\"url\": \"https://corrode.dev/blog/async\", \"title\": \"The State of Async Rust: Runtimes | corrode Rust Consulting\", \"publish_date\": \"2024-02-21\", \"excerpts\": [\"Through this series, I hope to address two different audiences:\\n* Newcomers to async Rust, seeking to get an overview of the current state of the ecosystem.\\n* Library maintainers a…\"]}, {\"url\": \"https://tokio.rs/\", \"title\": \"Tokio - An asynchronous Rust runtime\", \"publish_date\": null, \"excerpts\": [\"[tokio-logo](tokio.rs/)\\n\\n[Learn](tokio.rs/tokio/tutorial) [API Docs](docs.rs/tokio) [Blog](tokio.rs/blog/2026-10-06-tokioconf-2027-cfp)\\n\\n* * *\\n\\n[](twitter.com/tokio_rs) [](github.c…\"]}], \"warnings\": null, \"metadata\": null, \"session_id\": \"session_9bbd1740a684d0ea2cefa7a63dc5355c\"}", + "type": "text" + } + ], + "isError": false, + "structuredContent": { + "search_id": "search_9bbd1740a684d0ea2cefa7a63dc5355c", + "results": [ + { + "url": "https://rust-lang.github.io/async-book/08_ecosystem/00_chapter.html", + "title": "The Async Ecosystem - Asynchronous Programming in Rust", + "publish_date": null, + "excerpts": [ + "* Auto\n* Light\n* Rust\n* Coal\n* Navy\n* Ayu\n\nAsynchronous Programming in Rust\n\n \n\nThe Async Ecosystem\n\nRust currently provides only the bare essentials for writing async code. Import…" + ] + }, + { + "url": "https://corrode.dev/blog/async", + "title": "The State of Async Rust: Runtimes | corrode Rust Consulting", + "publish_date": "2024-02-21", + "excerpts": [ + "Through this series, I hope to address two different audiences:\n* Newcomers to async Rust, seeking to get an overview of the current state of the ecosystem.\n* Library maintainers a…" + ] + }, + { + "url": "https://tokio.rs/", + "title": "Tokio - An asynchronous Rust runtime", + "publish_date": null, + "excerpts": [ + "[tokio-logo](tokio.rs/)\n\n[Learn](tokio.rs/tokio/tutorial) [API Docs](docs.rs/tokio) [Blog](tokio.rs/blog/2026-10-06-tokioconf-2027-cfp)\n\n* * *\n\n[](twitter.com/tokio_rs) [](github.c…" + ] + } + ], + "warnings": null, + "metadata": null, + "session_id": "session_9bbd1740a684d0ea2cefa7a63dc5355c" + } + } +} \ No newline at end of file diff --git a/packages/coding-agent/test/fixtures/search/tavily.json b/packages/coding-agent/test/fixtures/search/tavily.json new file mode 100644 index 000000000..7003f17b6 --- /dev/null +++ b/packages/coding-agent/test/fixtures/search/tavily.json @@ -0,0 +1,30 @@ +{ + "query": "rust async runtime", + "follow_up_questions": null, + "answer": null, + "images": [], + "results": [ + { + "url": "https://doc.rust-lang.org/book/ch17-00-async-await.html", + "title": "Async and Await - The Rust Programming Language", + "content": "Rust's async/await syntax lets you write asynchronous code that looks synchronous.", + "score": 0.94, + "raw_content": null + }, + { + "url": "https://tokio.rs/tokio/tutorial", + "title": "Tutorial - Tokio", + "content": "Tokio is an asynchronous runtime for Rust: it provides the building blocks needed for writing network applications.", + "score": 0.91, + "raw_content": null + }, + { + "url": "https://rust-lang.github.io/async-book/", + "title": "Asynchronous Programming in Rust", + "content": "This book serves as high-level documentation for the async ecosystem.", + "score": 0.88, + "raw_content": null + } + ], + "response_time": 1.42 +} diff --git a/packages/coding-agent/test/fixtures/search/tinyfish.json b/packages/coding-agent/test/fixtures/search/tinyfish.json new file mode 100644 index 000000000..b0ccaa857 --- /dev/null +++ b/packages/coding-agent/test/fixtures/search/tinyfish.json @@ -0,0 +1,12 @@ +{ + "jsonrpc": "2.0", + "result": { + "content": [ + { + "type": "text", + "text": "{\"query\": \"rust async runtime\", \"results\": [{\"position\": 1, \"site_name\": \"rustasync.github.io\", \"snippet\": \"software.\\n- **runtime agnostic:** Runtime comes with minimal OS bindings out of the box, but switching to a different runtime is a matter of changing a single line.\\n- **await anywh…\", \"title\": \"runtime - Rust\", \"url\": \"https://rustasync.github.io/runtime/runtime\"}, {\"position\": 2, \"site_name\": \"docs.rs\", \"snippet\": \"and a reactor.\\n`Runtime` wraps any [`RuntimeKit`](traits/trait.RuntimeKit.html) and adds higher-level helpers such as domain-name resolution\", \"title\": \"Runtime in async_rs - Rust\", \"url\": \"https://docs.rs/async-rs/latest/async_rs/struct.Runtime.html\"}, {\"position\": 3, \"site_name\": \"docs.rs\", \"snippet\": \"ecosystem.\\n[`Runtime`](struct.Runtime.html) uses an `async-task` based scheduler with per-worker queues, global priority injectors, work stealing, and parking workers.\", \"title\": \"async_runtime - Rust\", \"url\": \"https://docs.rs/async-runtime/latest/async_runtime\"}], \"total_results\": 10, \"page\": 0}" + } + ] + }, + "id": 1 +} \ No newline at end of file diff --git a/packages/coding-agent/test/step-search-chain.test.ts b/packages/coding-agent/test/step-search-chain.test.ts new file mode 100644 index 000000000..3309187e0 --- /dev/null +++ b/packages/coding-agent/test/step-search-chain.test.ts @@ -0,0 +1,352 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it } from "vitest"; +import { findAdapter, KEYLESS_ADAPTERS } from "../src/step/search/adapters.ts"; +import { classifyFailure, parseRetryAfter } from "../src/step/search/errors.ts"; +import { buildSearchChain } from "../src/step/search/routing.ts"; +import { resetSearchCooldowns, runSearchChain } from "../src/step/search/scheduler.ts"; +import type { SearchAttempt, SearchCandidate } from "../src/step/search/types.ts"; + +const FIXTURES = join(import.meta.dirname, "fixtures", "search"); + +function fixture(name: string): unknown { + const recorded = JSON.parse(readFileSync(join(FIXTURES, `${name}.json`), "utf8")) as { + result?: unknown; + }; + // Fixtures keep the whole recorded response; adapters receive the payload the + // transport extracted from the JSON-RPC envelope. + return recorded.result ?? recorded; +} + +/** A random source that leaves the adapter order untouched. */ +const identityRandom = () => 0; + +function candidates(entries: readonly string[]): SearchCandidate[] { + return entries.map((id) => { + const adapter = findAdapter(id); + if (!adapter) throw new Error(`unknown adapter ${id}`); + return { + id: adapter.id, + tier: "keyless" as const, + endpoint: adapter.endpoint, + toolName: adapter.toolName, + transport: adapter.transport, + allowHandshake: true, + ...(adapter.headers ? { headers: adapter.headers } : {}), + }; + }); +} + +describe("search adapters", () => { + it.each(["exa", "firecrawl", "parallel", "tinyfish", "tavily"] as const)( + "normalizes a recorded %s response into hits", + (name) => { + const adapter = findAdapter(name); + expect(adapter).toBeDefined(); + const outcome = adapter!.parse(fixture(name)); + expect(outcome.hits.length).toBeGreaterThan(0); + for (const hit of outcome.hits) { + expect(hit.url).toMatch(/^https?:\/\//u); + expect(hit.url).not.toContain("…"); + } + expect(outcome.hits.length).toBeLessThanOrEqual(10); + }, + ); + + it("reads Firecrawl structured content before the text block", () => { + const outcome = findAdapter("firecrawl")!.parse(fixture("firecrawl")); + expect(outcome.hits[0]?.title).toBeTruthy(); + expect(outcome.hits[0]?.snippet).toBeTruthy(); + }); + + it("keeps Exa's published date and drops the N/A placeholder", () => { + const exa = findAdapter("exa")!; + const withDate = exa.parse({ + content: [ + { + type: "text", + text: "Title: Dated\nURL: https://example.com/a\nPublished: 2024-05-01T00:00:00Z\nHighlights:\nsome text", + }, + ], + }); + expect(withDate.hits[0]?.publishedAtMs).toBe(Date.parse("2024-05-01T00:00:00Z")); + + const withoutDate = exa.parse({ + content: [{ type: "text", text: "Title: Undated\nURL: https://example.com/b\nPublished: N/A" }], + }); + expect(withoutDate.hits[0]).not.toHaveProperty("publishedAtMs"); + }); + + it("detects Firecrawl's keyless quota message inside a 200 response", () => { + const outcome = findAdapter("firecrawl")!.parse({ + structuredContent: { success: true, data: { web: [] } }, + content: [{ type: "text", text: "You have reached your daily limit of free credits." }], + }); + // A quota message without hits must not look like an empty result set. + expect(outcome.hits).toHaveLength(0); + expect(outcome.text).toContain("daily limit"); + }); + + it("requests ten results from every provider that accepts a count", () => { + const exa = findAdapter("exa")!.buildRequest("q"); + expect(exa.args).toEqual({ query: "q", numResults: 10 }); + expect(findAdapter("firecrawl")!.buildRequest("q").args).toEqual({ query: "q", limit: 10 }); + expect(findAdapter("parallel")!.buildRequest("q").args).toEqual({ + objective: "q", + search_queries: ["q"], + }); + // Tavily is the only plain-HTTP provider; its body carries the search mode. + const tavily = findAdapter("tavily")!.buildRequest("q"); + expect(tavily.transport).toBe("http"); + expect(tavily.args).toEqual({ + query: "q", + search_depth: "basic", + chunks_per_source: 3, + max_results: 10, + }); + }); + + it("sends no credential to any free provider", () => { + for (const adapter of KEYLESS_ADAPTERS) { + const request = adapter.buildRequest("q"); + const headerNames = Object.keys(request.headers ?? {}).map((name) => name.toLowerCase()); + expect(headerNames).not.toContain("authorization"); + expect(headerNames).not.toContain("x-api-key"); + } + }); +}); + +describe("search failure classification", () => { + it("classifies a 200 response carrying an error body", () => { + expect(classifyFailure({ text: "web_search_exa error (401): Invalid API key", isError: true }).kind).toBe("auth"); + expect( + classifyFailure({ text: "You reached the daily keyless limit. daily cap", status: 200, isError: true }).kind, + ).toBe("quota"); + expect(classifyFailure({ text: "rate limit exceeded", status: 200, isError: true }).kind).toBe("rate_limit"); + }); + + it("recognizes every spelling of a daily ceiling", () => { + // Observed live: Tavily answers 429 with `daily_cap_reached`. + expect(classifyFailure({ status: 429, text: '{"error":{"code":"daily_cap_reached"}}' }).kind).toBe("quota"); + expect(classifyFailure({ text: "You reached the daily keyless Tavily limit.", isError: true }).kind).toBe( + "quota", + ); + expect(classifyFailure({ text: "daily limit reached", isError: true }).kind).toBe("quota"); + expect(classifyFailure({ text: "quota exceeded", isError: true }).kind).toBe("quota"); + }); + + it("classifies by status when the body says nothing useful", () => { + expect(classifyFailure({ status: 401, text: "" }).kind).toBe("auth"); + expect(classifyFailure({ status: 402, text: "" }).kind).toBe("quota"); + expect(classifyFailure({ status: 429, text: "" }).kind).toBe("rate_limit"); + expect(classifyFailure({ status: 503, text: "" }).kind).toBe("server"); + }); + + it("treats an unrecognized failure as unknown but keeps the provider text", () => { + const failure = classifyFailure({ status: 418, text: "teapot" }); + expect(failure.kind).toBe("unknown"); + expect(failure.detail).toBe("teapot"); + }); + + it("reports cancellation as cancelled, never as a provider outage", () => { + const failure = classifyFailure({ cancelled: true, networkError: "This operation was aborted" }); + expect(failure.kind).toBe("cancelled"); + }); + + it("parses both Retry-After forms", () => { + expect(parseRetryAfter("30")).toBe(30_000); + const at = Date.parse("2030-01-01T00:00:00Z"); + expect(parseRetryAfter("2030-01-01T00:00:00Z", at - 5_000)).toBe(5_000); + expect(parseRetryAfter(undefined)).toBeUndefined(); + }); +}); + +describe("search candidate chain", () => { + it("puts the paid tier first when a Step credential resolves", () => { + const chain = buildSearchChain({ + mode: "auto", + hasStepCredential: true, + stepEndpoint: "https://api.stepfun.com/v1/mcp/web_search/mcp", + random: identityRandom, + }); + expect(chain.candidates[0]).toMatchObject({ id: "stepfun", tier: "stepfun" }); + expect(chain.candidates).toHaveLength(6); + }); + + it("omits the paid tier entirely without a credential", () => { + const chain = buildSearchChain({ + mode: "auto", + hasStepCredential: false, + stepEndpoint: "https://api.stepfun.com/v1/mcp/web_search/mcp", + random: identityRandom, + }); + // Not "present then fails": the candidate is absent, so no doomed call is made. + expect(chain.candidates.map((entry) => entry.id)).not.toContain("stepfun"); + expect(chain.candidates).toHaveLength(5); + expect(chain.skippedPaidTier).toBe("no-step-credential"); + }); + + it("shuffles the free providers so traffic does not always hit the first one", () => { + const first = new Set(); + for (let seed = 0; seed < 40; seed += 1) { + const chain = buildSearchChain({ + mode: "auto", + hasStepCredential: false, + stepEndpoint: "https://api.stepfun.com/v1/mcp/web_search/mcp", + random: () => ((seed * 37) % 101) / 101, + }); + first.add(chain.candidates[0]!.id); + } + expect(first.size).toBeGreaterThan(1); + }); + + it("keeps every free provider in a shuffled chain", () => { + const chain = buildSearchChain({ + mode: "auto", + hasStepCredential: false, + stepEndpoint: "https://api.stepfun.com/v1/mcp/web_search/mcp", + random: () => 0.42, + }); + expect(new Set(chain.candidates.map((entry) => entry.id))).toEqual( + new Set(KEYLESS_ADAPTERS.map((adapter) => adapter.id)), + ); + }); + + it("builds a single-candidate chain for an explicit provider", () => { + const chain = buildSearchChain({ mode: "firecrawl", hasStepCredential: true, stepEndpoint: "x" }); + expect(chain.candidates.map((entry) => entry.id)).toEqual(["firecrawl"]); + }); + + it("disables search when configured off", () => { + expect( + buildSearchChain({ + mode: "off", + hasStepCredential: true, + stepEndpoint: "https://api.stepfun.com/v1/mcp/web_search/mcp", + }).disabled, + ).toBe(true); + }); +}); + +describe("search chain scheduling", () => { + beforeEach(() => { + resetSearchCooldowns(); + }); + + it("falls through to the next provider when one fails", async () => { + const tried: string[] = []; + const result = await runSearchChain({ + chain: { candidates: candidates(["exa", "firecrawl"]) }, + query: "q", + invoke: async (candidate) => { + tried.push(candidate.id); + if (candidate.id === "exa") { + return { ok: false, failure: { kind: "quota", detail: "daily limit" } } satisfies SearchAttempt; + } + return { ok: true, outcome: { hits: [{ url: "https://example.com" }] } } satisfies SearchAttempt; + }, + }); + expect(tried).toEqual(["exa", "firecrawl"]); + expect(result.provider).toBe("firecrawl"); + expect(result.degraded).toBe(true); + }); + + it("reports a paid-tier answer as not degraded", async () => { + const result = await runSearchChain({ + chain: { + candidates: [ + { + id: "stepfun", + tier: "stepfun", + endpoint: "https://api.stepfun.com/v1/mcp/web_search/mcp", + toolName: "web_search", + transport: "mcp", + allowHandshake: false, + }, + ], + }, + query: "q", + invoke: async () => ({ ok: true, outcome: { hits: [{ url: "https://example.com" }] } }), + }); + expect(result.degraded).toBe(false); + expect(result.tier).toBe("stepfun"); + }); + + it("stops immediately when the caller cancels", async () => { + const controller = new AbortController(); + let calls = 0; + await expect( + runSearchChain({ + chain: { candidates: candidates(["exa", "firecrawl", "tavily"]) }, + query: "q", + signal: controller.signal, + invoke: async () => { + calls += 1; + controller.abort(); + return { ok: false, failure: { kind: "cancelled", detail: "cancelled" } }; + }, + }), + ).rejects.toThrow(/cancelled/u); + // A cancelled kind must not advance: no other provider may be contacted. + expect(calls).toBe(1); + }); + + it("skips a candidate that is cooling down instead of calling it again", async () => { + const chain = { candidates: candidates(["exa", "firecrawl"]) }; + await expect( + runSearchChain({ + chain, + query: "q", + invoke: async () => ({ ok: false, failure: { kind: "rate_limit", detail: "slow down" } }), + }), + ).rejects.toThrow(/failed after 2/u); + + const tried: string[] = []; + const error = await runSearchChain({ + chain, + query: "q", + invoke: async (candidate) => { + tried.push(candidate.id); + return { ok: true, outcome: { hits: [{ url: "https://example.com" }] } }; + }, + }).catch((cause: unknown) => cause); + // Both providers are cooling down; the second call must not repeat them. + expect(tried).toEqual([]); + expect(String(error)).toContain("cooling down"); + expect(String(error)).toMatch(/retry in about \d+s/u); + }); + + it("lists every attempt and the skipped providers when nothing works", async () => { + const error = await runSearchChain({ + chain: { candidates: candidates(["exa", "firecrawl"]) }, + query: "q", + invoke: async (candidate) => ({ + ok: false, + failure: { kind: "server", detail: `${candidate.id} is down` }, + }), + }).catch((cause: unknown) => cause); + expect(String(error)).toContain("exa"); + expect(String(error)).toContain("firecrawl"); + expect(String(error)).toContain("is down"); + }); + + it("gives up when the shared budget is exhausted", async () => { + let clock = 0; + const tried: string[] = []; + const error = await runSearchChain({ + chain: { candidates: candidates(["exa", "firecrawl", "tavily"]) }, + query: "q", + budgetMs: 5_000, + now: () => clock, + invoke: async (candidate) => { + tried.push(candidate.id); + clock += 4_000; + return { ok: false, failure: { kind: "network", detail: "timeout" } }; + }, + }).catch((cause: unknown) => cause); + // 4s elapses on the first attempt, leaving 1s: below the 3s minimum, so the + // remaining providers are never contacted. + expect(tried).toEqual(["exa"]); + expect(String(error)).toContain("budget"); + }); +}); diff --git a/packages/coding-agent/test/step-search-config.test.ts b/packages/coding-agent/test/step-search-config.test.ts new file mode 100644 index 000000000..304f30c8a --- /dev/null +++ b/packages/coding-agent/test/step-search-config.test.ts @@ -0,0 +1,64 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { resolveSearchChainMode, SEARCH_PROVIDER_ENV } from "../src/step/search-web-tool.ts"; + +async function withConfigRoot(content: string | undefined, run: (env: Record) => Promise | T) { + const root = await mkdtemp(join(tmpdir(), "step-search-config-")); + try { + // `resolveStepConfigRoot` derives the config directory from the agent + // directory's parent, so pointing that at a temp path isolates the file. + const env: Record = { STEP_CODING_AGENT_DIR: join(root, "agent") }; + if (content !== undefined) await writeFile(join(root, "config.toml"), content, "utf8"); + return await run(env as Record); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +describe("search backend selection", () => { + it("defaults to auto when nothing is configured", async () => { + await withConfigRoot(undefined, (env) => { + expect(resolveSearchChainMode({ env })).toBe("auto"); + }); + }); + + it("reads the provider from config.toml", async () => { + await withConfigRoot('[search]\nprovider = "firecrawl"\n', (env) => { + expect(resolveSearchChainMode({ env })).toBe("firecrawl"); + }); + }); + + it("prefers the environment over config.toml", async () => { + await withConfigRoot('[search]\nprovider = "firecrawl"\n', (env) => { + expect(resolveSearchChainMode({ env: { ...env, [SEARCH_PROVIDER_ENV]: "tavily" } })).toBe("tavily"); + }); + }); + + it("prefers an explicit option over both", async () => { + await withConfigRoot('[search]\nprovider = "firecrawl"\n', (env) => { + expect(resolveSearchChainMode({ env: { ...env, [SEARCH_PROVIDER_ENV]: "tavily" }, provider: "off" })).toBe( + "off", + ); + }); + }); + + it("treats an unrecognized provider as auto instead of failing", async () => { + await withConfigRoot(undefined, (env) => { + // An older config or a removed provider must not disable search. + expect(resolveSearchChainMode({ env: { ...env, [SEARCH_PROVIDER_ENV]: "exaa" } })).toBe("auto"); + expect(resolveSearchChainMode({ env: { ...env, [SEARCH_PROVIDER_ENV]: "some-removed-provider" } })).toBe( + "auto", + ); + }); + }); + + it("still reports a malformed config file, which is not a value problem", async () => { + await withConfigRoot("this is not toml [[[\n", (env) => { + // An unreadable file is a real error; it is not the same as an + // unrecognized value, which degrades to auto. + expect(() => resolveSearchChainMode({ env })).toThrow(/Invalid Step config TOML/u); + }); + }); +}); diff --git a/packages/coding-agent/test/step-search-scenarios.test.ts b/packages/coding-agent/test/step-search-scenarios.test.ts new file mode 100644 index 000000000..012f927a6 --- /dev/null +++ b/packages/coding-agent/test/step-search-scenarios.test.ts @@ -0,0 +1,278 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { resetSearchCooldowns } from "../src/step/search/scheduler.ts"; +import { createSearchWebTool } from "../src/step/search-web-tool.ts"; + +/** + * End-to-end scenarios for the tool as a model actually calls it. + * + * These drive `createSearchWebTool().execute()` rather than the chain builder, + * so they cover the wiring between configuration, credential resolution, the + * ordered chain and rendering. + */ + +const originalFetch = globalThis.fetch; + +/** A provider answer in the shape the transport expects for each adapter. */ +function answerFor(url: string): Response { + if (url.includes("exa.ai")) { + return sseResult("Title: Exa hit\nURL: https://exa.example/1\nHighlights:\nexa snippet"); + } + if (url.includes("firecrawl")) { + return sseResult( + JSON.stringify({ + success: true, + data: { web: [{ url: "https://firecrawl.example/1", title: "FC", description: "fc snippet" }] }, + }), + ); + } + if (url.includes("parallel.ai")) { + return Response.json({ + jsonrpc: "2.0", + id: 1, + result: { + structuredContent: { + search_id: "s", + results: [{ url: "https://parallel.example/1", title: "P", excerpts: ["p snippet"] }], + }, + content: [{ type: "text", text: "{}" }], + }, + }); + } + if (url.includes("tinyfish")) { + return Response.json({ + jsonrpc: "2.0", + id: 1, + result: { + content: [ + { + type: "text", + text: JSON.stringify({ + results: [{ url: "https://tinyfish.example/1", title: "T", snippet: "t snippet" }], + }), + }, + ], + }, + }); + } + if (url.includes("tavily")) { + return Response.json({ results: [{ url: "https://tavily.example/1", title: "TV", content: "tv snippet" }] }); + } + return new Response("unexpected endpoint", { status: 500 }); +} + +function sseResult(text: string): Response { + return new Response( + `event: message\ndata: ${JSON.stringify({ jsonrpc: "2.0", id: 1, result: { content: [{ type: "text", text }] } })}\n\n`, + { + status: 200, + headers: { "content-type": "text/event-stream" }, + }, + ); +} + +function text(result: { content: Array<{ type: string; text?: string }> }): string { + return result.content + .filter((block) => block.type === "text") + .map((block) => block.text ?? "") + .join("\n"); +} + +function textHit(result: { content: Array<{ type: string; text?: string }> }): string | undefined { + return text(result).match(/\((https:\/\/[^)]+)\)/u)?.[1]; +} + +/** Record every endpoint the tool contacts, answering the way that provider does. */ +function installFetch(fail: (url: string) => Response | undefined = () => undefined): string[] { + const urls: string[] = []; + globalThis.fetch = (async (input: string | URL | Request) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + urls.push(url); + return fail(url) ?? answerFor(url); + }) as typeof fetch; + return urls; +} + +beforeEach(() => { + resetSearchCooldowns(); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +describe("search_web scenarios", () => { + it("auto without a Step credential uses the free providers only", async () => { + const root = await mkdtemp(join(tmpdir(), "step-search-scenario-")); + const urls = installFetch(); + try { + let stepCalls = 0; + const tool = createSearchWebTool( + { env: {}, authPath: join(root, "missing.json"), random: () => 0.99 }, + async () => { + stepCalls += 1; + return { structuredContent: { results: [] } }; + }, + ); + + const result = await tool.execute("call", { query: "q" }, undefined, undefined, undefined as never); + + // The paid endpoint cannot succeed without a credential, so it is absent + // from the chain rather than attempted-and-failed. + expect(stepCalls).toBe(0); + expect(urls).toHaveLength(1); + expect(result.details?.degraded).toBe(true); + expect(textHit(result)).toBeTruthy(); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("auto with a Step credential tries the paid tier first", async () => { + const urls = installFetch(); + const order: string[] = []; + const tool = createSearchWebTool({ env: {}, apiKey: "search-key", random: () => 0.99 }, async () => { + order.push("stepfun"); + return { structuredContent: { results: [{ title: "Paid", url: "https://paid.example/1" }] } }; + }); + + const result = await tool.execute("call", { query: "q" }, undefined, undefined, undefined as never); + + expect(order).toEqual(["stepfun"]); + // A paid answer must not touch a free provider. + expect(urls).toHaveLength(0); + expect(result.details?.provider).toBe("stepfun"); + expect(result.details?.degraded).toBe(false); + expect(textHit(result)).toBe("https://paid.example/1"); + }); + + it("falls back to free providers when the paid tier reports an exhausted quota", async () => { + installFetch(); + const tool = createSearchWebTool({ env: {}, apiKey: "search-key", random: () => 0.99 }, async () => ({ + isError: true, + content: "quota exceeded for this billing period", + })); + + const result = await tool.execute("call", { query: "q" }, undefined, undefined, undefined as never); + + expect(result.details?.provider).not.toBe("stepfun"); + expect(result.details?.degraded).toBe(true); + expect(textHit(result)).toBeTruthy(); + }); + + it("falls back when the paid tier is unreachable", async () => { + installFetch(); + const tool = createSearchWebTool({ env: {}, apiKey: "search-key", random: () => 0.99 }, async () => { + throw new Error("connect ECONNREFUSED 127.0.0.1:443"); + }); + + const result = await tool.execute("call", { query: "q" }, undefined, undefined, undefined as never); + expect(result.details?.degraded).toBe(true); + }); + + it("pins a configured provider and never falls back to another", async () => { + const urls = installFetch((url) => (url.includes("exa.ai") ? new Response("gone", { status: 503 }) : undefined)); + const tool = createSearchWebTool({ env: {}, provider: "exa" }); + + await expect(tool.execute("call", { query: "q" }, undefined, undefined, undefined as never)).rejects.toThrow( + /exa/u, + ); + // Only the pinned provider is contacted, even though four others are healthy. + expect(urls.every((url) => url.includes("exa.ai"))).toBe(true); + }); + + it("refuses the call when search is configured off", async () => { + const urls = installFetch(); + const tool = createSearchWebTool({ env: {}, provider: "off" }); + + await expect(tool.execute("call", { query: "q" }, undefined, undefined, undefined as never)).rejects.toThrow( + /disabled/u, + ); + expect(urls).toHaveLength(0); + }); + + it("treats an unrecognized provider setting as auto", async () => { + const urls = installFetch(); + const tool = createSearchWebTool({ env: { STEPCODE_SEARCH_PROVIDER: "not-a-provider" }, random: () => 0.99 }); + + const result = await tool.execute("call", { query: "q" }, undefined, undefined, undefined as never); + + expect(urls.length).toBeGreaterThan(0); + expect(textHit(result)).toBeTruthy(); + }); + + it("fails with an empty query before contacting anything", async () => { + const urls = installFetch(); + const tool = createSearchWebTool({ env: {} }); + + await expect(tool.execute("call", { query: " " }, undefined, undefined, undefined as never)).rejects.toThrow( + /must not be empty/u, + ); + expect(urls).toHaveLength(0); + }); + + it("walks every free provider and names each failure when all of them fail", async () => { + const attempted: string[] = []; + installFetch((url) => { + attempted.push(url); + return new Response("upstream exploded", { status: 503 }); + }); + const tool = createSearchWebTool({ env: {}, random: () => 0.99 }); + + const error = await tool + .execute("call", { query: "q" }, undefined, undefined, undefined as never) + .then(() => undefined) + .catch((cause: unknown) => cause); + + expect(String(error)).toMatch(/failed after 5 attempt/u); + expect(attempted).toHaveLength(5); + // The report must name each provider so a user can tell which one is broken. + for (const id of ["exa", "firecrawl", "parallel", "tavily", "tinyfish"]) { + expect(String(error)).toContain(id); + } + }); + + it("skips a provider that is cooling down on the next call", async () => { + const attempted: string[] = []; + const failAll = true; + installFetch((url) => { + attempted.push(url); + return failAll ? new Response("rate limited", { status: 429, headers: { "retry-after": "60" } }) : undefined; + }); + const tool = createSearchWebTool({ env: {}, random: () => 0.99 }); + + await tool.execute("call-1", { query: "q" }, undefined, undefined, undefined as never).catch(() => undefined); + expect(attempted).toHaveLength(5); + + // Every provider is now cooling down: the next call must not repeat them. + const again: string[] = []; + installFetch((url) => { + again.push(url); + return undefined; + }); + const error = await tool + .execute("call-2", { query: "q" }, undefined, undefined, undefined as never) + .then(() => undefined) + .catch((cause: unknown) => cause); + + expect(again).toHaveLength(0); + expect(String(error)).toContain("cooling down"); + }); + + it("accepts a second search after the budget-limited first one", async () => { + // Two sequential calls must both work: shared state must not leak a + // failure into a later, independent search. + installFetch(); + const tool = createSearchWebTool({ env: {}, random: () => 0.99 }); + + const first = await tool.execute("c1", { query: "one" }, undefined, undefined, undefined as never); + const second = await tool.execute("c2", { query: "two" }, undefined, undefined, undefined as never); + + expect(textHit(first)).toBeTruthy(); + expect(textHit(second)).toBeTruthy(); + expect(first.details?.query).toBe("one"); + expect(second.details?.query).toBe("two"); + }); +}); diff --git a/packages/coding-agent/test/step-search-transport.test.ts b/packages/coding-agent/test/step-search-transport.test.ts new file mode 100644 index 000000000..0335dcd60 --- /dev/null +++ b/packages/coding-agent/test/step-search-transport.test.ts @@ -0,0 +1,305 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { findAdapter } from "../src/step/search/adapters.ts"; +import { callSearchProvider } from "../src/step/search/transport.ts"; + +const originalFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function sse(body: unknown): string { + return `event: message\ndata: ${JSON.stringify(body)}\n\n`; +} + +function json(body: unknown): string { + return JSON.stringify(body); +} + +const exa = findAdapter("exa")!; + +function stubFetch(handler: (url: string, init: RequestInit) => Response | Promise): string[] { + const urls: string[] = []; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + urls.push(url); + return handler(url, init ?? {}); + }) as typeof fetch; + return urls; +} + +function searchOnce(options: { allowHandshake?: boolean; signal?: AbortSignal } = {}) { + return callSearchProvider({ + request: exa.buildRequest("query"), + parse: exa.parse, + timeoutMs: 5_000, + allowHandshake: options.allowHandshake ?? true, + serverName: "exa", + ...(options.signal ? { signal: options.signal } : {}), + }); +} + +describe("search transport", () => { + it("calls the provider once, statelessly, without a credential", async () => { + const urls = stubFetch( + () => + new Response( + sse({ + jsonrpc: "2.0", + id: 1, + result: { + content: [ + { type: "text", text: "Title: Doc\nURL: https://example.com/doc\nHighlights:\nbody text" }, + ], + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ); + + const attempt = await searchOnce(); + expect(attempt.ok).toBe(true); + if (!attempt.ok) return; + expect(attempt.outcome.hits[0]).toMatchObject({ url: "https://example.com/doc", title: "Doc" }); + // One POST: no initialize, no tools/list. + expect(urls).toEqual([exa.endpoint]); + }); + + it("sends a user-agent and no authorization header", async () => { + let headers: Record = {}; + stubFetch((_url, init) => { + headers = (init.headers ?? {}) as Record; + return new Response(sse({ jsonrpc: "2.0", id: 1, result: { content: [] } }), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + }); + + await searchOnce(); + const names = Object.keys(headers).map((name) => name.toLowerCase()); + expect(names).toContain("user-agent"); + expect(names).not.toContain("authorization"); + }); + + it("reads a provider error delivered inside a 200 response", async () => { + stubFetch( + () => + new Response( + sse({ + jsonrpc: "2.0", + id: 1, + result: { + isError: true, + content: [{ type: "text", text: "You reached your daily limit of free credits." }], + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ); + + const attempt = await searchOnce(); + // A 200 must not be reported as "no results": the chain has to advance. + expect(attempt.ok).toBe(false); + if (attempt.ok) return; + expect(attempt.failure.kind).toBe("quota"); + }); + + it("retries through a handshake when the provider refuses a stateless call", async () => { + const urls = stubFetch(() => + Response.json( + { jsonrpc: "2.0", id: 1, error: { code: -32601, message: "Method not found" } }, + { status: 404 }, + ), + ); + + const attempt = await searchOnce(); + // The SDK client cannot reach the stubbed endpoint either, so this ends as a + // network failure, but the important part is that a second call was made. + expect(attempt.ok).toBe(false); + expect(urls.length).toBeGreaterThan(1); + }); + + it("does not retry a rejected credential", async () => { + const urls = stubFetch( + () => + new Response( + sse({ + jsonrpc: "2.0", + id: 1, + result: { isError: true, content: [{ type: "text", text: "Unauthorized: invalid API key" }] }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ); + + const attempt = await searchOnce(); + expect(attempt.ok).toBe(false); + if (attempt.ok) return; + expect(attempt.failure.kind).toBe("auth"); + // One attempt only: retrying a bad credential would double the latency. + expect(urls.length).toBe(1); + }); + + it("reports a caller cancellation as cancelled, not as a provider outage", async () => { + const controller = new AbortController(); + stubFetch(() => { + controller.abort(); + return Promise.reject(new DOMException("This operation was aborted", "AbortError")); + }); + + const attempt = await searchOnce({ signal: controller.signal }); + expect(attempt.ok).toBe(false); + if (attempt.ok) return; + expect(attempt.failure.kind).toBe("cancelled"); + }); + + it("rejects an oversized response instead of buffering it", async () => { + stubFetch( + () => + new Response( + json({ + jsonrpc: "2.0", + id: 1, + result: { content: [{ type: "text", text: "x".repeat(2 * 1024 * 1024) }] }, + }), + { + status: 200, + headers: { "content-type": "application/json" }, + }, + ), + ); + + const attempt = await searchOnce(); + expect(attempt.ok).toBe(false); + if (attempt.ok) return; + expect(attempt.failure.kind).toBe("parse"); + expect(attempt.failure.detail).toContain("exceeded"); + }); + + it("returns real hits from a handshake retry, whose content arrives as one string", async () => { + // The SDK client joins content blocks into a single string. Handing that + // straight to an adapter parses as "no results" and ends the chain on a + // false empty answer, so the transport must rebuild the block list first. + let statelessRefused = false; + stubFetch((_url, init) => { + // The SDK also opens a GET stream; only POST bodies carry a JSON-RPC method. + const body = init.body ? (JSON.parse(String(init.body)) as { id?: number; method?: string }) : undefined; + if (!body?.method) return new Response(null, { status: 202 }); + if (!statelessRefused) { + statelessRefused = true; + return Response.json( + { jsonrpc: "2.0", id: 1, error: { code: -32601, message: "Method not found" } }, + { status: 404 }, + ); + } + // The SDK needs a session id from initialize before it will call a tool. + const headers = { "content-type": "application/json", "mcp-session-id": "test-session" }; + switch (body.method) { + case "initialize": + return Response.json( + { + jsonrpc: "2.0", + id: body.id ?? 1, + result: { + protocolVersion: "2025-06-18", + capabilities: { tools: {} }, + serverInfo: { name: "exa", version: "test" }, + }, + }, + { headers }, + ); + case "tools/list": + return Response.json( + { + jsonrpc: "2.0", + id: body.id ?? 2, + result: { + tools: [ + { + name: "web_search_exa", + inputSchema: { type: "object", properties: { query: { type: "string" } } }, + }, + ], + }, + }, + { headers }, + ); + case "tools/call": + return Response.json( + { + jsonrpc: "2.0", + id: body.id ?? 3, + result: { + content: [ + { + type: "text", + text: "Title: Retried\nURL: https://example.com/retried\nHighlights:\nbody", + }, + ], + }, + }, + { headers }, + ); + default: + return new Response(null, { status: 202, headers: { "mcp-session-id": "test-session" } }); + } + }); + + const attempt = await searchOnce(); + expect(attempt.ok).toBe(true); + if (!attempt.ok) return; + expect(attempt.outcome.hits[0]).toMatchObject({ url: "https://example.com/retried", title: "Retried" }); + }); + + it("posts Tavily's search body and parses its plain JSON results", async () => { + const tavily = findAdapter("tavily")!; + let body: unknown; + let headers: Record = {}; + stubFetch((_url, init) => { + body = JSON.parse(String(init.body)); + headers = (init.headers ?? {}) as Record; + return Response.json({ + results: [ + { url: "https://example.com/t", title: "T", content: "tavily snippet" }, + { url: "https://example.com/u", title: "U", content: "second" }, + ], + }); + }); + + const attempt = await callSearchProvider({ + request: tavily.buildRequest("query"), + parse: tavily.parse, + timeoutMs: 5_000, + allowHandshake: false, + serverName: "tavily", + }); + + expect(attempt.ok).toBe(true); + if (!attempt.ok) return; + expect(attempt.outcome.hits[0]).toMatchObject({ url: "https://example.com/t", snippet: "tavily snippet" }); + expect(body).toMatchObject({ query: "query", search_depth: "basic", max_results: 10 }); + // The access-mode header is what keeps Tavily keyless; without it the call 401s. + expect(headers["X-Tavily-Access-Mode"]).toBe("keyless"); + }); + + it("parses a plain JSON response as well as an event stream", async () => { + stubFetch( + () => + new Response( + json({ + jsonrpc: "2.0", + id: 1, + result: { + content: [{ type: "text", text: "Title: Plain\nURL: https://example.com/plain\nHighlights:\nx" }], + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ); + + const attempt = await searchOnce(); + expect(attempt.ok).toBe(true); + if (!attempt.ok) return; + expect(attempt.outcome.hits[0]?.url).toBe("https://example.com/plain"); + }); +}); diff --git a/packages/coding-agent/test/step-search-web.test.ts b/packages/coding-agent/test/step-search-web.test.ts index b14fe7c44..8512f57b9 100644 --- a/packages/coding-agent/test/step-search-web.test.ts +++ b/packages/coding-agent/test/step-search-web.test.ts @@ -1,7 +1,8 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it } from "vitest"; +import { resetSearchCooldowns } from "../src/step/search/scheduler.ts"; import { createSearchWebTool, resolveSearchWebApiKey, @@ -17,6 +18,12 @@ function text(result: { content: Array<{ type: string; text?: string }> }): stri } describe("Step search_web tool", () => { + beforeEach(() => { + // Cooldowns are process-wide so a new session does not repeat a call that + // already failed; tests need a clean slate per case. + resetSearchCooldowns(); + }); + it("resolves endpoint origins and preserves explicit paths", () => { expect(resolveSearchWebServerUrl("https://search.example.com")).toBe( "https://search.example.com/v1/mcp/web_search/mcp", @@ -117,28 +124,31 @@ describe("Step search_web tool", () => { it("calls the remote tool with a bearer header and compacts results", async () => { const calls: SearchWebInvocation[] = []; - const tool = createSearchWebTool({ url: "https://search.example.com", apiKey: "search-key" }, async (input) => { - calls.push(input); - return { - structuredContent: { - results: [ - { - position: 1, - title: "Example result", - url: "https://example.com", - snippet: "short snippet", - content: "full page content must not reach the model", - }, - { - position: 2, - title: "Long result", - url: "https://example.org", - snippet: "x".repeat(450), - }, - ], - }, - }; - }); + const tool = createSearchWebTool( + { url: "https://search.example.com", apiKey: "search-key", env: {}, provider: "step" }, + async (input) => { + calls.push(input); + return { + structuredContent: { + results: [ + { + position: 1, + title: "Example result", + url: "https://example.com", + snippet: "short snippet", + content: "full page content must not reach the model", + }, + { + position: 2, + title: "Long result", + url: "https://example.org", + snippet: "x".repeat(450), + }, + ], + }, + }; + }, + ); const result = await tool.execute( "call-1", @@ -164,6 +174,8 @@ describe("Step search_web tool", () => { query: "current information", resultCount: 2, urls: ["https://example.com", "https://example.org"], + provider: "stepfun", + degraded: false, }); }); @@ -187,7 +199,7 @@ describe("Step search_web tool", () => { "utf8", ); let serverUrl = ""; - const tool = createSearchWebTool({ env: {}, authPath }, async (input) => { + const tool = createSearchWebTool({ env: {}, authPath, provider: "step" }, async (input) => { serverUrl = input.serverUrl; return { structuredContent: { results: [] } }; }); @@ -200,27 +212,50 @@ describe("Step search_web tool", () => { } }); - it("passes cancellation to the remote invoker", async () => { + it("forwards the caller signal to the remote invoker", async () => { const controller = new AbortController(); - controller.abort(); let receivedSignal: AbortSignal | undefined; - const tool = createSearchWebTool({ apiKey: "search-key" }, async (input) => { + const tool = createSearchWebTool({ apiKey: "search-key", env: {}, provider: "step" }, async (input) => { receivedSignal = input.signal; - return { structuredContent: { results: [] } }; + // Abort while the call is in flight; the invoker still sees the caller's + // own signal, not a copy. + controller.abort(); + throw new Error("aborted"); }); - await tool.execute("call-2", { query: "query" }, controller.signal, undefined, undefined as never); + await expect( + tool.execute("call-2", { query: "query" }, controller.signal, undefined, undefined as never), + ).rejects.toThrow(/cancelled/u); expect(receivedSignal).toBe(controller.signal); }); + it("does not contact a provider when the caller already cancelled", async () => { + const controller = new AbortController(); + controller.abort(); + let called = false; + const tool = createSearchWebTool({ apiKey: "search-key", env: {}, provider: "step" }, async () => { + called = true; + return { structuredContent: { results: [] } }; + }); + + await expect( + tool.execute("call-2b", { query: "query" }, controller.signal, undefined, undefined as never), + ).rejects.toThrow(/cancelled/u); + // A cancelled call must not spend money or hit a free provider. + expect(called).toBe(false); + }); + it("fails before making a request when no credential is available", async () => { const root = await mkdtemp(join(tmpdir(), "step-search-web-")); try { let called = false; - const tool = createSearchWebTool({ env: {}, authPath: join(root, "missing-auth.json") }, async () => { - called = true; - return { structuredContent: { results: [] } }; - }); + const tool = createSearchWebTool( + { env: {}, authPath: join(root, "missing-auth.json"), provider: "step" }, + async () => { + called = true; + return { structuredContent: { results: [] } }; + }, + ); await expect( tool.execute("call-3", { query: "query" }, undefined, undefined, undefined as never), @@ -232,7 +267,7 @@ describe("Step search_web tool", () => { }); it("turns remote MCP errors into tool failures that name the billed endpoint", async () => { - const tool = createSearchWebTool({ apiKey: "search-key" }, async () => ({ + const tool = createSearchWebTool({ apiKey: "search-key", env: {}, provider: "step" }, async () => ({ isError: true, content: "502 Bad Gateway", })); @@ -243,7 +278,7 @@ describe("Step search_web tool", () => { }); it("preserves transport failures as diagnostic tool errors", async () => { - const tool = createSearchWebTool({ apiKey: "search-key" }, async () => { + const tool = createSearchWebTool({ apiKey: "search-key", env: {}, provider: "step" }, async () => { throw new Error("transport unavailable"); }); @@ -251,4 +286,86 @@ describe("Step search_web tool", () => { tool.execute("call-5", { query: "query" }, undefined, undefined, undefined as never), ).rejects.toThrow("transport unavailable"); }); + + it("searches with a free provider when no Step credential exists", async () => { + const root = await mkdtemp(join(tmpdir(), "step-search-free-")); + const urls: string[] = []; + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + urls.push(url); + return new Response( + `event: message\ndata: ${JSON.stringify({ + jsonrpc: "2.0", + id: 1, + result: { + content: [{ type: "text", text: "Title: Free\nURL: https://free.example/doc\nHighlights:\nsnippet" }], + }, + })}\n\n`, + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }) as typeof fetch; + + try { + let stepCalls = 0; + // A random source near 1 leaves the declared order untouched, so the + // first free candidate is the Exa-shaped fixture below. + const tool = createSearchWebTool( + { env: {}, authPath: join(root, "missing-auth.json"), random: () => 0.99 }, + async () => { + stepCalls += 1; + return { structuredContent: { results: [] } }; + }, + ); + + const result = await tool.execute("call-free", { query: "query" }, undefined, undefined, undefined as never); + + // The paid endpoint is never contacted: no credential means it cannot succeed. + expect(stepCalls).toBe(0); + expect(urls.length).toBe(1); + expect(text(result)).toContain("[Free](https://free.example/doc)"); + expect(result.details).toMatchObject({ provider: "exa", degraded: true, resultCount: 1 }); + } finally { + globalThis.fetch = originalFetch; + await rm(root, { recursive: true, force: true }); + } + }); + + it("falls back to a free provider when the paid tier reports exhausted quota", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async () => + new Response( + `event: message\ndata: ${JSON.stringify({ + jsonrpc: "2.0", + id: 1, + result: { + content: [ + { type: "text", text: "Title: Fallback\nURL: https://free.example/fallback\nHighlights:\nx" }, + ], + }, + })}\n\n`, + { status: 200, headers: { "content-type": "text/event-stream" } }, + )) as typeof fetch; + + try { + const tool = createSearchWebTool( + { env: {}, apiKey: "search-key", provider: "auto", random: () => 0.99 }, + async () => ({ isError: true, content: "You reached your daily Plan search limit" }), + ); + + const result = await tool.execute( + "call-fallback", + { query: "query" }, + undefined, + undefined, + undefined as never, + ); + + expect(text(result)).toContain("https://free.example/fallback"); + // The answer must say it came from a free provider, not the paid tier. + expect(result.details).toMatchObject({ provider: "exa", degraded: true }); + } finally { + globalThis.fetch = originalFetch; + } + }); }); From f2fb1dda5880ff6e46f928ae5d8515cf0742571d Mon Sep 17 00:00:00 2001 From: liuedcson <332840681+liuedcson@users.noreply.github.com> Date: Sat, 21 Nov 2026 11:46:16 +0800 Subject: [PATCH 2/2] fix(search): keep the paid endpoint's result numbering and URL-less entries The rewrite rendered results strictly by array index and dropped entries without a URL. Both changed what the model sees for the paid endpoint: - a provider-assigned `position` was replaced with the array index, so a numbering gap was silently renumbered; - an entry without a URL was removed instead of being shown under its position and title. Restore both, and correct the `provider` option JSDoc: the environment variable takes precedence over config.toml, not the other way round. The added case pins the numbering, the URL-less entry, and the index fallback so a later refactor cannot quietly drop them again. --- .../coding-agent/src/step/search-web-tool.ts | 23 ++++++++++------ .../coding-agent/src/step/search/types.ts | 11 +++++++- .../coding-agent/test/step-search-web.test.ts | 26 +++++++++++++++++++ 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/packages/coding-agent/src/step/search-web-tool.ts b/packages/coding-agent/src/step/search-web-tool.ts index 9711fc2d6..531db357d 100644 --- a/packages/coding-agent/src/step/search-web-tool.ts +++ b/packages/coding-agent/src/step/search-web-tool.ts @@ -82,8 +82,8 @@ export interface SearchWebToolOptions { /** Injected environment, kept public so credential resolution is testable. */ env?: Record; /** - * Backend selection. Defaults to `[search] provider` in config.toml, then - * `STEPCODE_SEARCH_PROVIDER`, then `auto`. + * Backend selection. `STEPCODE_SEARCH_PROVIDER` takes precedence over + * `[search] provider` in config.toml; when neither is set, `auto` is used. */ provider?: SearchChainMode; /** Injected randomness for the free-provider order; kept testable. */ @@ -344,22 +344,24 @@ function readStepFunOutcome(result: SearchWebMcpResult): SearchOutcome { return text ? { hits: [], text } : { hits: [] }; } const hits: SearchHit[] = []; - for (const item of items) { + items.forEach((item, index) => { + if (hits.length >= SEARCH_WEB_RESULT_COUNT) return; const record = isRecord(item) ? item : {}; const url = normalizeOptionalText(record.url); - if (!url) continue; const title = normalizeOptionalText(record.title); const snippet = shortenText( normalizeOptionalText(record.snippet) ?? normalizeOptionalText(record.content) ?? "", MAX_SNIPPET_CHARS, ); hits.push({ - url, + ...(url ? { url } : {}), ...(title ? { title } : {}), ...(snippet ? { snippet } : {}), + // The provider's own numbering wins when present; otherwise the array + // order is the position, exactly as the previous renderer did. + position: typeof record.position === "number" ? record.position : index + 1, }); - if (hits.length >= SEARCH_WEB_RESULT_COUNT) break; - } + }); return { hits }; } @@ -387,7 +389,12 @@ function renderSearchResults(query: string, result: SearchChainResult): AgentToo const content = result.outcome.hits .map((hit, index) => { - const heading = `${index + 1}. [${hit.title ?? hit.url}](${hit.url})`; + // Same shape as before: a linked heading when a URL exists, otherwise + // the position and title alone, so an entry without a URL is still shown. + const position = hit.position ?? index + 1; + const heading = hit.url + ? `${position}. [${hit.title ?? hit.url}](${hit.url})` + : `${position}. ${hit.title ?? "(untitled)"}`; return hit.snippet ? `${heading}\n ${shortenText(hit.snippet, MAX_SNIPPET_CHARS)}` : heading; }) .join("\n\n"); diff --git a/packages/coding-agent/src/step/search/types.ts b/packages/coding-agent/src/step/search/types.ts index 749e83079..a70b3adbc 100644 --- a/packages/coding-agent/src/step/search/types.ts +++ b/packages/coding-agent/src/step/search/types.ts @@ -1,8 +1,17 @@ /** One normalized search result, independent of which provider produced it. */ export interface SearchHit { - url: string; + /** + * Result URL. Optional because the paid endpoint can return an entry without + * one; those entries are still shown, under their position and title. + */ + url?: string; title?: string; snippet?: string; + /** + * Provider-assigned position, when it supplies one. Rendering falls back to + * the array index so a provider ordering gap is not renumbered. + */ + position?: number; /** Publication time in milliseconds since the Unix epoch, when the provider reports one. */ publishedAtMs?: number; } diff --git a/packages/coding-agent/test/step-search-web.test.ts b/packages/coding-agent/test/step-search-web.test.ts index 8512f57b9..babbd2bc0 100644 --- a/packages/coding-agent/test/step-search-web.test.ts +++ b/packages/coding-agent/test/step-search-web.test.ts @@ -287,6 +287,32 @@ describe("Step search_web tool", () => { ).rejects.toThrow("transport unavailable"); }); + it("keeps the paid endpoint's own numbering and shows entries without a URL", async () => { + const tool = createSearchWebTool({ apiKey: "search-key", env: {}, provider: "step" }, async () => ({ + structuredContent: { + results: [ + // The provider skips 2: the reported numbering must survive. + { position: 1, title: "First", url: "https://example.com/1" }, + { position: 3, title: "Third", url: "https://example.com/3" }, + { position: 4, title: "No link" }, + { title: "Unnumbered" }, + ], + }, + })); + + const result = await tool.execute("call-pos", { query: "q" }, undefined, undefined, undefined as never); + const body = text(result); + + expect(body).toContain("1. [First](https://example.com/1)"); + expect(body).toContain("3. [Third](https://example.com/3)"); + // An entry without a URL is still reported, not silently dropped. + expect(body).toContain("4. No link"); + // A missing position falls back to the array index, matching the old renderer. + expect(body).toContain("4. Unnumbered"); + expect(result.details.urls).toEqual(["https://example.com/1", "https://example.com/3"]); + expect(result.details.resultCount).toBe(4); + }); + it("searches with a free provider when no Step credential exists", async () => { const root = await mkdtemp(join(tmpdir(), "step-search-free-")); const urls: string[] = [];