From 6016991dad3e8d42d036dac6db153494250a891b Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 16 Nov 2026 16:52:26 +0100 Subject: [PATCH] fix(vcs): stop recording expected GitHub and scan outcomes as errors - vendor-http: a 3xx answer ends its attempt span Ok. GitHub's job log endpoint always answers 302 to the signed download, which made ~29k error spans in two days. - GithubAppClient.getCommit: a blocked repository (451, or 403 with a block) fails as VcsRepositoryBlockedError, like the listing calls. - DependencyScanner: a blocked, gone or uninstalled repository is a RepositoryUnreachable outcome, and a tree too large to list records its failed scan row and returns TreeTooLarge, both without failing the span. The plan cache holds null for a truncated tree instead of failing inside the cache span. - VcsSource*NotFoundError move to the domain with a 404, so a wrong guess by the agent's code tools is an anticipated outcome. - lockfiles: strip a byte order mark before parsing, read bun.lock comments as Bun does, and name the yarn.lock entry a parse failure is about. --- lib/lockfiles/src/internal/decode.ts | 9 ++ lib/lockfiles/src/parse.test.ts | 35 +++++++ lib/lockfiles/src/parse.ts | 6 +- lib/lockfiles/src/parsers/bun.ts | 4 +- lib/lockfiles/src/parsers/yarn.ts | 25 ++++- .../dependency-scan/DependencyScanner.test.ts | 60 ++++++++++-- .../dependency-scan/DependencyScanner.ts | 94 +++++++++++++------ .../integrations/vcs/VcsSourceService.ts | 18 +--- .../vcs/vendor/github/GithubAppClient.ts | 6 +- .../GithubAppClient.normal-states.test.ts | 28 +++++- .../services/integrations/vendor-http.test.ts | 44 +++++++++ .../src/services/integrations/vendor-http.ts | 26 ++++- .../anticipated-error-identifiers.ts | 3 + packages/domain/src/http/vcs.ts | 21 +++++ 14 files changed, 312 insertions(+), 67 deletions(-) diff --git a/lib/lockfiles/src/internal/decode.ts b/lib/lockfiles/src/internal/decode.ts index 6482862a66..8d26d13d07 100644 --- a/lib/lockfiles/src/internal/decode.ts +++ b/lib/lockfiles/src/internal/decode.ts @@ -13,6 +13,9 @@ const parseError = (path: string, what: string) => (cause: unknown) => /** Matches a JSON string literal (kept) or a trailing comma before `]`/`}` (dropped). */ const STRING_OR_TRAILING_COMMA = /"(?:[^"\\]|\\.)*"|,(\s*[\]}])/g +/** Matches a JSON string literal (kept) or a `//` or `/* *\/` comment (dropped). */ +const STRING_OR_COMMENT = /("(?:[^"\\]|\\.)*")|\/\/[^\n]*|\/\*[\s\S]*?\*\//g + export const stripTrailingCommas = (text: string): string => replaceWith( text, @@ -20,6 +23,12 @@ export const stripTrailingCommas = (text: string): string => (match: string, closer: string | undefined) => closer ?? match, ) +/** JSONC as Bun reads it: comments and trailing commas. Comments go first, so `, // x\n}` loses its comma too. */ +export const stripJsonc = (text: string): string => + stripTrailingCommas( + replaceWith(text, STRING_OR_COMMENT, (_: string, literal: string | undefined) => literal ?? ""), + ) + export const decodeJson = ( schema: Schema.Decoder, path: string, diff --git a/lib/lockfiles/src/parse.test.ts b/lib/lockfiles/src/parse.test.ts index 1ad3fd4ca3..5ea8e59f95 100644 --- a/lib/lockfiles/src/parse.test.ts +++ b/lib/lockfiles/src/parse.test.ts @@ -81,6 +81,18 @@ describe("unsupported and malformed input", () => { expect(error._tag).toBe("@maple/lockfiles/LockfileParseError") expect(error.path).toBe(path) }) + + it("names the yarn.lock entry the parser could not read, without echoing its range", async () => { + const missingVersion = await failure( + "yarn.lock", + 'lodash@https://user:secret@host/lodash.tgz:\n resolved "x"\n', + ) + const notAnEntry = await failure("yarn.lock", 'a@^1:\n version "1.0.0"\n<<<<<<< HEAD\n') + expect([missingVersion.message, notAnEntry.message]).toEqual([ + "yarn.lock entry for lodash at line 1 has no version line", + "yarn.lock line 3 is neither an entry nor a field: <<<<<<< HEAD", + ]) + }) }) describe("npm lockfile details", () => { @@ -263,6 +275,29 @@ describe("npm lockfile details", () => { expect(parsed.warnings).toEqual(["skipped 1 non-registry packages (git, tarball or local)"]) }) + it("bun.lock: a byte order mark and comments, which Bun reads, parse; comment text inside strings is kept", async () => { + const lock = `\uFEFF{ + // written by hand + "lockfileVersion": 1, /* the text format */ + "workspaces": { "": { "name": "app", "dependencies": { "a": "^1" } } }, + "packages": { + "a": ["a@1.0.0", "", {}, "sha512-//x/*y*/"], // trailing + }, +}` + const parsed = await parse("bun.lock", lock) + expect(summary(parsed)).toEqual(["a@1.0.0:direct"]) + }) + + it("package-lock.json with a byte order mark parses", async () => { + const lock = { + lockfileVersion: 3, + packages: { "": { dependencies: { a: "^1" } }, "node_modules/a": { version: "1.0.0" } }, + } + expect(summary(await parse("package-lock.json", `\uFEFF${JSON.stringify(lock)}`))).toEqual([ + "a@1.0.0:direct", + ]) + }) + it("deduplicates by name and version, keeping the strongest relation", async () => { const lock = { lockfileVersion: 3, diff --git a/lib/lockfiles/src/parse.ts b/lib/lockfiles/src/parse.ts index efa0af03b5..e159c983b8 100644 --- a/lib/lockfiles/src/parse.ts +++ b/lib/lockfiles/src/parse.ts @@ -1,4 +1,4 @@ -import { Effect } from "effect" +import { Effect, String as Str } from "effect" import { LockfileParseError, LockfileUnsupportedError } from "./errors.ts" import { dedupePackages } from "./internal/collect.ts" import { parseBunLock } from "./parsers/bun.ts" @@ -35,7 +35,9 @@ export const parseLockfile = Effect.fn("lockfiles.parseLockfile")(function* ( if (kind === "bun-lockb") { return yield* new LockfileUnsupportedError({ path: input.path, message: BUN_LOCKB_UNSUPPORTED }) } - const output = yield* PARSERS[kind](input) + // An editor's byte order mark is not part of any format, and JSON.parse refuses it. + const content = Str.startsWith("\uFEFF")(input.content) ? Str.slice(1)(input.content) : input.content + const output = yield* PARSERS[kind]({ ...input, content }) return { path: input.path, kind: output.kind, diff --git a/lib/lockfiles/src/parsers/bun.ts b/lib/lockfiles/src/parsers/bun.ts index d18a14df13..db58b1ba16 100644 --- a/lib/lockfiles/src/parsers/bun.ts +++ b/lib/lockfiles/src/parsers/bun.ts @@ -1,6 +1,6 @@ import { Effect, Schema, Array as Arr, Record as Rec, String as Str } from "effect" import { isRegistryVersion, pkg, splitNameAt } from "../internal/collect.ts" -import { decodeJson, stripTrailingCommas } from "../internal/decode.ts" +import { decodeJson, stripJsonc } from "../internal/decode.ts" import type { LockedPackage, LockfileInput, Relation } from "../types.ts" import type { ParserOutput } from "./types.ts" @@ -48,7 +48,7 @@ const namesOf = (workspaces: ReadonlyArray): DepNames => ({ }) export const parseBunLock = Effect.fn("lockfiles.parseBunLock")(function* (input: LockfileInput) { - const lock = yield* decodeJson(BunLock, input.path, stripTrailingCommas(input.content)) + const lock = yield* decodeJson(BunLock, input.path, stripJsonc(input.content)) const workspaces = Rec.values(lock.workspaces ?? {}) const all = namesOf(workspaces) const byWorkspaceName = new Map( diff --git a/lib/lockfiles/src/parsers/yarn.ts b/lib/lockfiles/src/parsers/yarn.ts index aff4a65725..9b6fc404d1 100644 --- a/lib/lockfiles/src/parsers/yarn.ts +++ b/lib/lockfiles/src/parsers/yarn.ts @@ -1,4 +1,4 @@ -import { Effect, Array as Arr, Record as Rec, String as Str } from "effect" +import { Effect, Option, Array as Arr, Record as Rec, String as Str } from "effect" import { isRecord, isRegistryVersion, pkg, splitNameAt } from "../internal/collect.ts" import { parseYamlText } from "../internal/decode.ts" import { readPackageJsons, type ManifestDeps } from "../internal/manifests.ts" @@ -15,6 +15,9 @@ interface Specifier { interface Block { readonly specifiers: ReadonlyArray readonly version: string | undefined + /** 1-based, so a parse error can point at the entry. */ + readonly line: number + readonly header: string } const unquote = (s: string): string => Str.replace(/^"(.*)"$/, "$1")(Str.trim(s)) @@ -30,10 +33,15 @@ const VERSION_LINE = /^ {2}"?version"?:? +"?([^"\s]+)"?\s*$/ /** yarn v1: unindented `"a@^1", a@^1.2:` headers, each followed by an indented `version "1.2.3"`. */ const parseV1Blocks = (content: string): ReadonlyArray => - Arr.reduce(Str.split(/\r?\n/)(content), Arr.empty(), (blocks, line) => { + Arr.reduce(Str.split(/\r?\n/)(content), Arr.empty(), (blocks, line, index) => { if (line === "" || Str.startsWith("#")(line)) return blocks if (!/^\s/.test(line)) { - blocks.push({ specifiers: parseSpecifiers(line), version: undefined }) + blocks.push({ + specifiers: parseSpecifiers(line), + version: undefined, + line: index + 1, + header: line, + }) return blocks } const version = VERSION_LINE.exec(line)?.[1] @@ -120,10 +128,17 @@ const berryResult = Effect.fn(function* (input: LockfileInput, manifest: Manifes const v1Result = Effect.fn(function* (input: LockfileInput, manifest: ManifestDeps) { const blocks = parseV1Blocks(input.content) - if (Arr.some(blocks, (b) => b.specifiers.length === 0 || b.version === undefined)) { + const broken = Arr.findFirst(blocks, (b) => b.specifiers.length === 0 || b.version === undefined) + if (Option.isSome(broken)) { + const { line, header, specifiers } = broken.value + // Says which shape the parser missed; a range is never echoed, since a tarball URL can carry a token. + const first = specifiers[0] return yield* new LockfileParseError({ path: input.path, - message: "yarn.lock has an entry without specifiers or version", + message: + first === undefined + ? `yarn.lock line ${line} is neither an entry nor a field: ${Str.slice(0, 40)(header)}` + : `yarn.lock entry for ${first.name} at line ${line} has no version line`, }) } return parseYarnV1(blocks, manifest) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts index b80569cbe0..1ff165f4b2 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts @@ -1,6 +1,13 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" -import { GitCommitSha, VcsInstallation, VcsRepo } from "@maple/domain/http" +import { + GitCommitSha, + VcsInstallation, + VcsInstallationGoneError, + VcsRepo, + VcsRepositoryBlockedError, + VcsRepoUnavailableError, +} from "@maple/domain/http" import { Deferred, Effect, @@ -107,6 +114,8 @@ interface World { head: GitCommitSha | null files: Map truncated: boolean + /** What the provider answers a ref lookup with instead of the head. */ + refusal: VcsRepositoryBlockedError | VcsRepoUnavailableError | VcsInstallationGoneError | undefined osvDown: boolean /** Where the branch head moves while the tree is read, as a concurrent push would. */ headAfterTree: GitCommitSha | null | undefined @@ -131,6 +140,7 @@ const world = (files: Record): World => ({ ]), ), truncated: false, + refusal: undefined, osvDown: false, headAfterTree: undefined, stallBlobs: undefined, @@ -142,9 +152,11 @@ const world = (files: Record): World => ({ const source = (state: World) => ({ resolveRef: () => - Effect.sync(() => { + Effect.suspend(() => { state.calls.resolveRef += 1 - return Option.fromNullOr(state.head) + return state.refusal === undefined + ? Effect.succeed(Option.fromNullOr(state.head)) + : Effect.fail(state.refusal) }), fetchTreeFiles: () => Effect.sync(() => { @@ -575,7 +587,7 @@ describe("DependencyScanner", () => { }) it.effect( - "fails a scan whose tree listing was cut short instead of fixing the unlisted lockfiles' findings", + "records a scan whose tree listing was cut short as failed instead of fixing the unlisted lockfiles' findings", () => { const state = world({ "package-lock.json": packageLock("4.17.15") }) return Effect.gen(function* () { @@ -584,10 +596,11 @@ describe("DependencyScanner", () => { state.head = sha("b") state.truncated = true state.files.delete("package-lock.json") - const error = yield* Effect.flip( - scanner.scan(installation, repository, { trigger: "schedule" }), - ) - assert.include(error.message, "too large") + const outcome = yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual(outcome._tag, "TreeTooLarge") + if (outcome._tag !== "TreeTooLarge") return + assert.deepStrictEqual([outcome.scan.status, outcome.scan.commitSha], ["failed", sha("b")]) + assert.include(outcome.scan.error ?? "", "too large") assert.isTrue(Arr.every(yield* findingsOf, (finding) => finding.status === "open")) }).pipe(Effect.provide(scannerFor(state))) }, @@ -599,17 +612,44 @@ describe("DependencyScanner", () => { const scanner = yield* DependencyScanner const store = yield* DependencyScanStore state.truncated = true - yield* Effect.flip(scanner.scan(installation, repository, { trigger: "push" })) + const first = yield* scanner.scan(installation, repository, { trigger: "push" }) + assert.strictEqual(first._tag, "TreeTooLarge") state.head = sha("b") const again = yield* scanner.scan(installation, repository, { trigger: "push" }) assert.strictEqual(again._tag, "Unchanged") assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 1) // Scan now records its own result, so the page sees the scan it asked for. - yield* Effect.flip(scanner.scan(installation, repository, { trigger: "manual" })) + yield* scanner.scan(installation, repository, { trigger: "manual" }) assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 2) }).pipe(Effect.provide(scannerFor(state))) }) + it.effect("answers a repository the provider blocks or lost as unreachable, recording no scan", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + const refusals = [ + new VcsRepositoryBlockedError({ message: "Repository access blocked", status: 451 }), + new VcsRepoUnavailableError({ message: "Not Found" }), + new VcsInstallationGoneError({ message: "Not Found" }), + ] + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const store = yield* DependencyScanStore + const reasons = yield* Effect.forEach(refusals, (refusal) => { + state.refusal = refusal + return scanner + .scan(installation, repository, { trigger: "schedule" }) + .pipe( + Effect.map((outcome) => + outcome._tag === "RepositoryUnreachable" ? outcome.reason : outcome._tag, + ), + ) + }) + assert.deepStrictEqual(reasons, ["blocked", "unavailable", "installation_gone"]) + assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 0) + assert.strictEqual(state.calls.fetchTreeFiles, 0) + }).pipe(Effect.provide(scannerFor(state))) + }) + it.effect("reads a yarn lockfile's manifests but never one too large to be a package.json", () => { const state = world({ "yarn.lock": '"lodash@4.17.15":\n version "4.17.15"\n', diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.ts index e1a1e3f8e7..6cedc0e00f 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.ts @@ -16,7 +16,18 @@ import { } from "@maple/domain/http" import { type PackageAdvisory, PackageEcosystem, packageKey } from "@maple/domain/package-advisories" import { parseLockfile, unsupportedReason } from "@maple/lockfiles" -import { Cause, Clock, Context, Effect, Layer, Option, Schema, String as Str, Array as Arr } from "effect" +import { + Cause, + Clock, + Context, + Effect, + Layer, + Option, + Result, + Schema, + String as Str, + Array as Arr, +} from "effect" import type { VcsProviderClient, VcsTreeFile, @@ -48,11 +59,11 @@ import { RESCAN_AFTER_MS } from "./production" const PARSE_CACHE_BUCKET = "lockfile-packages" /** Part of the parse cache key: bump it when a parser fix should reach lockfiles already cached. */ -const PARSER_VERSION = 2 +const PARSER_VERSION = 3 const PARSE_CACHE_TTL_SECONDS = 7 * 24 * 60 * 60 const PLAN_CACHE_BUCKET = "dependency-scan-plan" /** Part of the plan cache key: bump it when lockfile or manifest selection changes. */ -const PLAN_VERSION = 1 +const PLAN_VERSION = 2 /** Larger lockfiles are recorded as too large rather than read into the Worker's memory. */ export const MAX_LOCKFILE_BYTES = 8 * 1024 * 1024 /** Two lockfiles at a time keeps a monorepo's largest pair well inside a Worker's memory. */ @@ -107,8 +118,8 @@ const planOf = (files: ReadonlyArray): ScanPlan => { } } -/** Internal: a cut-short tree listing, never cached as a plan. */ -class TreeTruncated extends Schema.TaggedError()("@maple/dependency-scan/TreeTruncated", {}) {} +/** A commit's plan, or null when its tree is too large to list: a commit's tree never changes. */ +const CachedPlan = Schema.NullOr(ScanPlan) export class DependencyScanError extends Schema.TaggedError()( "@maple/dependency-scan/DependencyScanError", @@ -140,6 +151,15 @@ export type DependencyScanOutcome = | { readonly _tag: "Unchanged" } /** The tracked branch has no commit yet. */ | { readonly _tag: "EmptyRepository" } + /** The provider will not show the repository: blocked, gone, or the installation removed. */ + | { readonly _tag: "RepositoryUnreachable"; readonly reason: UnreachableReason } + /** The tree is too large to list; the failed scan row says so. */ + | { readonly _tag: "TreeTooLarge"; readonly scan: DependencyScanRow } + +type UnreachableReason = "blocked" | "unavailable" | "installation_gone" + +const unreachable = (reason: UnreachableReason) => () => + Effect.succeed(Result.fail(reason)) export interface DependencyScannerApi { readonly scan: ( @@ -342,24 +362,13 @@ export class DependencyScanner extends Context.Service - tree.truncated - ? Effect.fail(new TreeTruncated()) - : Effect.succeed(planOf(tree.files)), - ), - ), - ) - .pipe( - Effect.map(({ value }) => Option.some(value)), - Effect.catchTag("@maple/dependency-scan/TreeTruncated", () => - Effect.succeed(Option.none()), - ), + .pipe(Effect.map((tree) => (tree.truncated ? null : planOf(tree.files)))), ) + .pipe(Effect.map(({ value }) => Option.fromNullOr(value))) const matchesOf = ( reads: ReadonlyArray, @@ -431,12 +440,33 @@ export class DependencyScanner extends Context.Service Result.succeed(head)), + Effect.catchTags({ + "@maple/http/errors/VcsRepositoryBlockedError": + unreachable("blocked"), + "@maple/http/errors/VcsRepoUnavailableError": + unreachable("unavailable"), + "@maple/http/errors/VcsInstallationGoneError": + unreachable("installation_gone"), + }), + Effect.mapError(scanError(`Could not resolve ${ref}`)), + ) + : Result.succeed(Option.some(request.commitSha)) + if (Result.isFailure(resolved)) { + yield* Effect.annotateCurrentSpan({ + "maple.depscan.outcome": "repository_unreachable", + "maple.depscan.skip_reason": resolved.failure, + }) + return { + _tag: "RepositoryUnreachable", + reason: resolved.failure, + } satisfies DependencyScanOutcome + } + const headOpt = resolved.success if (Option.isNone(headOpt)) return { _tag: "EmptyRepository" } satisfies DependencyScanOutcome const commitSha = headOpt.value @@ -592,13 +622,17 @@ export class DependencyScanner extends Context.Service()( - "@maple/backend/vcs/VcsSourceRepositoryNotFoundError", - { repository: Schema.String, message: Schema.String }, -) {} - -export class VcsSourceFileNotFoundError extends Schema.TaggedError()( - "@maple/backend/vcs/VcsSourceFileNotFoundError", - { repository: Schema.String, path: Schema.String, ref: Schema.String, message: Schema.String }, -) {} +export { VcsSourceFileNotFoundError, VcsSourceRefNotFoundError, VcsSourceRepositoryNotFoundError } /** * Everything that can go wrong resolving *which* repository to talk to, before @@ -55,11 +50,6 @@ type VcsRepositoryScopedError = type VcsSourceError = VcsRepositoryScopedError | VcsSourceFileNotFoundError -export class VcsSourceRefNotFoundError extends Schema.TaggedError()( - "@maple/backend/vcs/VcsSourceRefNotFoundError", - { repository: Schema.String, ref: Schema.String, message: Schema.String }, -) {} - /** A commit the sandbox can check out: the exact SHA a ref names, and how to clone it. */ export interface RepositoryCheckout { readonly provider: VcsRepo["provider"] diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts index ac4577f940..162547d9d2 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts @@ -1233,7 +1233,11 @@ export class GithubAppClient extends Context.Service()( return Option.none() return yield* failure(response, "Get commit", "repository") } - if (!isOk(response)) return yield* failure(response, "Get commit", "repository") + // A blocked repository fails as the anticipated domain error, so the span stays out of error tracking. + if (!isOk(response)) + return yield* failure(response, "Get commit", "repository").pipe( + Effect.mapError(toRepositoryListingError), + ) const json = yield* parseJson(response, "Get commit") return yield* decodeCommit(json).pipe( Effect.mapBoth({ diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/__tests__/GithubAppClient.normal-states.test.ts b/packages/backend/src/services/integrations/vcs/vendor/github/__tests__/GithubAppClient.normal-states.test.ts index ebb03b1e08..bac6c84720 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/__tests__/GithubAppClient.normal-states.test.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/__tests__/GithubAppClient.normal-states.test.ts @@ -119,7 +119,33 @@ describe("GithubAppClient responses that are normal states", () => { return Effect.gen(function* () { const client = yield* GithubAppClient const error = yield* Effect.flip(client.getCommit("42", "octo", "shop", "main")) - assert.deepStrictEqual([error.status, error.scope], [409, "repository"]) + assert.deepStrictEqual( + [error._tag, error.status, error._tag === "@maple/backend/vcs/GithubAppError" && error.scope], + ["@maple/backend/vcs/GithubAppError", 409, "repository"], + ) + }).pipe(Effect.provide(layer)) + }) + + it.effect("fails a commit lookup in a blocked repository as the repository being blocked", () => { + const blocked = (status: number) => + new Response( + '{"message":"Repository access blocked","block":{"reason":"dmca","created_at":"2025-08-26T19:06:05Z"}}', + { status }, + ) + const layer = clientLayer([tokenResponse(), blocked(451), blocked(403)], []) + return Effect.gen(function* () { + const client = yield* GithubAppClient + const legal = yield* Effect.flip(client.getCommit("42", "octo", "shop", "main")) + const tos = yield* Effect.flip(client.getCommit("42", "octo", "shop", "main")) + assert.deepStrictEqual( + [legal._tag, legal.status, tos._tag, tos.status], + [ + "@maple/http/errors/VcsRepositoryBlockedError", + 451, + "@maple/http/errors/VcsRepositoryBlockedError", + 403, + ], + ) }).pipe(Effect.provide(layer)) }) diff --git a/packages/backend/src/services/integrations/vendor-http.test.ts b/packages/backend/src/services/integrations/vendor-http.test.ts index 0cc126ef48..e176ba41cb 100644 --- a/packages/backend/src/services/integrations/vendor-http.test.ts +++ b/packages/backend/src/services/integrations/vendor-http.test.ts @@ -431,4 +431,48 @@ describe("VendorHttp", () => { }), ) }) + + it.effect("ends a 3xx attempt's span Ok and still fails the call, while a 4xx span fails", () => { + const spans: Array = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const fake = VendorFake.make({ + routes: [ + { + path: "/v1/logs", + respond: () => + VendorFake.text("", 302, { location: "https://blobs.acme.test/log?sig=1" }), + }, + { path: "/v1/gone", respond: () => VendorFake.text("", 404) }, + ], + }) + return withClient(fake, (client) => + Effect.gen(function* () { + const redirect = yield* Effect.flip(client.text({ path: "/v1/logs" })) + const gone = yield* Effect.flip(client.text({ path: "/v1/gone" })) + return [redirect.status, gone.status] + }).pipe(Effect.withTracer(tracer)), + ).pipe( + Effect.map((statuses) => { + assert.deepStrictEqual(statuses, [302, 404]) + const outcomes = Arr.map(spans, (span) => { + const attributes = Rec.fromEntries([...span.attributes]) + return [ + attributes["http.response.status_code"], + span.status._tag === "Ended" && span.status.exit._tag, + attributes["error.type"], + ] + }) + assert.deepStrictEqual(outcomes, [ + [302, "Success", undefined], + [404, "Failure", "not_found"], + ]) + }), + ) + }) }) diff --git a/packages/backend/src/services/integrations/vendor-http.ts b/packages/backend/src/services/integrations/vendor-http.ts index fdb559db5b..1793275bc9 100644 --- a/packages/backend/src/services/integrations/vendor-http.ts +++ b/packages/backend/src/services/integrations/vendor-http.ts @@ -3,7 +3,18 @@ * transient retries, `retry-after` on a 429, no redirects, paging, and a Client span that records * only the URL path. Each client maps {@link VendorHttpError} to its own errors through `toError`. */ -import { Clock, Duration, Effect, Option, Redacted, Schedule, Schema, Stream, String as Str } from "effect" +import { + Clock, + Duration, + Effect, + Option, + Redacted, + Result, + Schedule, + Schema, + Stream, + String as Str, +} from "effect" import { FetchHttpClient, HttpClient, HttpClientRequest, type HttpClientResponse } from "effect/http" import { parseDeltaSeconds, parseRetryAfterSeconds, VENDOR_TRANSIENT_RETRY } from "./vendor-retry" @@ -150,6 +161,9 @@ const isTransient = (error: VendorHttpError) => error.kind === "timeout" || (error.kind === "upstream" && (error.status === undefined || error.status >= 500)) +const isRedirect = (error: VendorHttpError) => + error.status !== undefined && error.status >= 300 && error.status < 400 + const decodeJsonText = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown)) export const make = (options: VendorHttpOptions): VendorHttpClient => { @@ -259,7 +273,14 @@ export const make = (options: VendorHttpOptions): VendorHttpClient => { duration: request.timeout ?? options.timeout, orElse: () => Effect.fail(fail("timeout", "timed out", request)), }), - Effect.tapError((error) => Effect.annotateCurrentSpan("error.type", error.kind)), + Effect.tapError((error) => + isRedirect(error) + ? Effect.void + : Effect.annotateCurrentSpan("error.type", error.kind), + ), + // A 3xx is a complete answer the caller may follow by hand, so its span ends Ok. + Effect.map(Result.succeed), + Effect.catchIf(isRedirect, (error) => Effect.succeed(Result.fail(error))), Effect.withSpan("VendorHttp.request", { kind: "client", attributes: { @@ -273,6 +294,7 @@ export const make = (options: VendorHttpOptions): VendorHttpClient => { : { "maple.vendor.operation": request.operation }), }, }), + Effect.flatMap(Effect.fromResult), ), ), ) diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index c2203c6b71..cf7673ec6f 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -5,6 +5,9 @@ // contracts outside production so worker startup imports only this literal. export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ + "@maple/backend/vcs/VcsSourceFileNotFoundError", + "@maple/backend/vcs/VcsSourceRefNotFoundError", + "@maple/backend/vcs/VcsSourceRepositoryNotFoundError", "@maple/http/ai-sessions/AiSessionTooLargeError", "@maple/http/ai-triage/AiTriageForbiddenError", "@maple/http/ai-triage/AiTriageValidationError", diff --git a/packages/domain/src/http/vcs.ts b/packages/domain/src/http/vcs.ts index 9b614ec773..ff40845ebf 100644 --- a/packages/domain/src/http/vcs.ts +++ b/packages/domain/src/http/vcs.ts @@ -860,3 +860,24 @@ export class OAuthStatePersistenceError extends Schema.TaggedError()( + "@maple/backend/vcs/VcsSourceRepositoryNotFoundError", + { repository: Schema.String, message: Schema.String }, + { httpApiStatus: 404 }, +) {} + +export class VcsSourceFileNotFoundError extends Schema.TaggedError()( + "@maple/backend/vcs/VcsSourceFileNotFoundError", + { repository: Schema.String, path: Schema.String, ref: Schema.String, message: Schema.String }, + { httpApiStatus: 404 }, +) {} + +export class VcsSourceRefNotFoundError extends Schema.TaggedError()( + "@maple/backend/vcs/VcsSourceRefNotFoundError", + { repository: Schema.String, ref: Schema.String, message: Schema.String }, + { httpApiStatus: 404 }, +) {}