Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions lib/lockfiles/src/internal/decode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,22 @@ const parseError = (path: string, what: string) => (cause: unknown) =>
/** Matches a JSON string literal (kept) or a trailing comma before `]`/`}` (dropped). */
const STRING_OR_TRAILING_COMMA = /"(?:[^"\\]|\\.)*"|,(\s*[\]}])/g

/** Matches a JSON string literal (kept) or a `//` or `/* *\/` comment (dropped). */
const STRING_OR_COMMENT = /("(?:[^"\\]|\\.)*")|\/\/[^\n]*|\/\*[\s\S]*?\*\//g

export const stripTrailingCommas = (text: string): string =>
replaceWith(
text,
STRING_OR_TRAILING_COMMA,
(match: string, closer: string | undefined) => closer ?? match,
)

/** JSONC as Bun reads it: comments and trailing commas. Comments go first, so `, // x\n}` loses its comma too. */
export const stripJsonc = (text: string): string =>
stripTrailingCommas(
replaceWith(text, STRING_OR_COMMENT, (_: string, literal: string | undefined) => literal ?? ""),
)

export const decodeJson = <T>(
schema: Schema.Decoder<T>,
path: string,
Expand Down
35 changes: 35 additions & 0 deletions lib/lockfiles/src/parse.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,18 @@ describe("unsupported and malformed input", () => {
expect(error._tag).toBe("@maple/lockfiles/LockfileParseError")
expect(error.path).toBe(path)
})

it("names the yarn.lock entry the parser could not read, without echoing its range", async () => {
const missingVersion = await failure(
"yarn.lock",
'lodash@https://user:secret@host/lodash.tgz:\n resolved "x"\n',
)
const notAnEntry = await failure("yarn.lock", 'a@^1:\n version "1.0.0"\n<<<<<<< HEAD\n')
expect([missingVersion.message, notAnEntry.message]).toEqual([
"yarn.lock entry for lodash at line 1 has no version line",
"yarn.lock line 3 is neither an entry nor a field: <<<<<<< HEAD",
])
})
})

describe("npm lockfile details", () => {
Expand Down Expand Up @@ -263,6 +275,29 @@ describe("npm lockfile details", () => {
expect(parsed.warnings).toEqual(["skipped 1 non-registry packages (git, tarball or local)"])
})

it("bun.lock: a byte order mark and comments, which Bun reads, parse; comment text inside strings is kept", async () => {
const lock = `\uFEFF{
// written by hand
"lockfileVersion": 1, /* the text format */
"workspaces": { "": { "name": "app", "dependencies": { "a": "^1" } } },
"packages": {
"a": ["a@1.0.0", "", {}, "sha512-//x/*y*/"], // trailing
},
}`
const parsed = await parse("bun.lock", lock)
expect(summary(parsed)).toEqual(["a@1.0.0:direct"])
})

it("package-lock.json with a byte order mark parses", async () => {
const lock = {
lockfileVersion: 3,
packages: { "": { dependencies: { a: "^1" } }, "node_modules/a": { version: "1.0.0" } },
}
expect(summary(await parse("package-lock.json", `\uFEFF${JSON.stringify(lock)}`))).toEqual([
"a@1.0.0:direct",
])
})

it("deduplicates by name and version, keeping the strongest relation", async () => {
const lock = {
lockfileVersion: 3,
Expand Down
6 changes: 4 additions & 2 deletions lib/lockfiles/src/parse.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Effect } from "effect"
import { Effect, String as Str } from "effect"
import { LockfileParseError, LockfileUnsupportedError } from "./errors.ts"
import { dedupePackages } from "./internal/collect.ts"
import { parseBunLock } from "./parsers/bun.ts"
Expand Down Expand Up @@ -35,7 +35,9 @@ export const parseLockfile = Effect.fn("lockfiles.parseLockfile")(function* (
if (kind === "bun-lockb") {
return yield* new LockfileUnsupportedError({ path: input.path, message: BUN_LOCKB_UNSUPPORTED })
}
const output = yield* PARSERS[kind](input)
// An editor's byte order mark is not part of any format, and JSON.parse refuses it.
const content = Str.startsWith("\uFEFF")(input.content) ? Str.slice(1)(input.content) : input.content
const output = yield* PARSERS[kind]({ ...input, content })
return {
path: input.path,
kind: output.kind,
Expand Down
4 changes: 2 additions & 2 deletions lib/lockfiles/src/parsers/bun.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Effect, Schema, Array as Arr, Record as Rec, String as Str } from "effect"
import { isRegistryVersion, pkg, splitNameAt } from "../internal/collect.ts"
import { decodeJson, stripTrailingCommas } from "../internal/decode.ts"
import { decodeJson, stripJsonc } from "../internal/decode.ts"
import type { LockedPackage, LockfileInput, Relation } from "../types.ts"
import type { ParserOutput } from "./types.ts"

Expand Down Expand Up @@ -48,7 +48,7 @@ const namesOf = (workspaces: ReadonlyArray<Workspace>): DepNames => ({
})

export const parseBunLock = Effect.fn("lockfiles.parseBunLock")(function* (input: LockfileInput) {
const lock = yield* decodeJson(BunLock, input.path, stripTrailingCommas(input.content))
const lock = yield* decodeJson(BunLock, input.path, stripJsonc(input.content))
const workspaces = Rec.values(lock.workspaces ?? {})
const all = namesOf(workspaces)
const byWorkspaceName = new Map(
Expand Down
25 changes: 20 additions & 5 deletions lib/lockfiles/src/parsers/yarn.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Effect, Array as Arr, Record as Rec, String as Str } from "effect"
import { Effect, Option, Array as Arr, Record as Rec, String as Str } from "effect"
import { isRecord, isRegistryVersion, pkg, splitNameAt } from "../internal/collect.ts"
import { parseYamlText } from "../internal/decode.ts"
import { readPackageJsons, type ManifestDeps } from "../internal/manifests.ts"
Expand All @@ -15,6 +15,9 @@ interface Specifier {
interface Block {
readonly specifiers: ReadonlyArray<Specifier>
readonly version: string | undefined
/** 1-based, so a parse error can point at the entry. */
readonly line: number
readonly header: string
}

const unquote = (s: string): string => Str.replace(/^"(.*)"$/, "$1")(Str.trim(s))
Expand All @@ -30,10 +33,15 @@ const VERSION_LINE = /^ {2}"?version"?:? +"?([^"\s]+)"?\s*$/

/** yarn v1: unindented `"a@^1", a@^1.2:` headers, each followed by an indented `version "1.2.3"`. */
const parseV1Blocks = (content: string): ReadonlyArray<Block> =>
Arr.reduce(Str.split(/\r?\n/)(content), Arr.empty<Block>(), (blocks, line) => {
Arr.reduce(Str.split(/\r?\n/)(content), Arr.empty<Block>(), (blocks, line, index) => {
if (line === "" || Str.startsWith("#")(line)) return blocks
if (!/^\s/.test(line)) {
blocks.push({ specifiers: parseSpecifiers(line), version: undefined })
blocks.push({
specifiers: parseSpecifiers(line),
version: undefined,
line: index + 1,
header: line,
})
return blocks
}
const version = VERSION_LINE.exec(line)?.[1]
Expand Down Expand Up @@ -120,10 +128,17 @@ const berryResult = Effect.fn(function* (input: LockfileInput, manifest: Manifes

const v1Result = Effect.fn(function* (input: LockfileInput, manifest: ManifestDeps) {
const blocks = parseV1Blocks(input.content)
if (Arr.some(blocks, (b) => b.specifiers.length === 0 || b.version === undefined)) {
const broken = Arr.findFirst(blocks, (b) => b.specifiers.length === 0 || b.version === undefined)
if (Option.isSome(broken)) {
const { line, header, specifiers } = broken.value
// Says which shape the parser missed; a range is never echoed, since a tarball URL can carry a token.
const first = specifiers[0]
return yield* new LockfileParseError({
path: input.path,
message: "yarn.lock has an entry without specifiers or version",
message:
first === undefined
? `yarn.lock line ${line} is neither an entry nor a field: ${Str.slice(0, 40)(header)}`
: `yarn.lock entry for ${first.name} at line ${line} has no version line`,
})
}
return parseYarnV1(blocks, manifest)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
import { afterEach, assert, describe, it } from "@effect/vitest"
import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache"
import { GitCommitSha, VcsInstallation, VcsRepo } from "@maple/domain/http"
import {
GitCommitSha,
VcsInstallation,
VcsInstallationGoneError,
VcsRepo,
VcsRepositoryBlockedError,
VcsRepoUnavailableError,
} from "@maple/domain/http"
import {
Deferred,
Effect,
Expand Down Expand Up @@ -107,6 +114,8 @@ interface World {
head: GitCommitSha | null
files: Map<string, { readonly sha: string; readonly content: string; readonly size?: number }>
truncated: boolean
/** What the provider answers a ref lookup with instead of the head. */
refusal: VcsRepositoryBlockedError | VcsRepoUnavailableError | VcsInstallationGoneError | undefined
osvDown: boolean
/** Where the branch head moves while the tree is read, as a concurrent push would. */
headAfterTree: GitCommitSha | null | undefined
Expand All @@ -131,6 +140,7 @@ const world = (files: Record<string, string>): World => ({
]),
),
truncated: false,
refusal: undefined,
osvDown: false,
headAfterTree: undefined,
stallBlobs: undefined,
Expand All @@ -142,9 +152,11 @@ const world = (files: Record<string, string>): World => ({

const source = (state: World) => ({
resolveRef: () =>
Effect.sync(() => {
Effect.suspend(() => {
state.calls.resolveRef += 1
return Option.fromNullOr(state.head)
return state.refusal === undefined
? Effect.succeed(Option.fromNullOr(state.head))
: Effect.fail(state.refusal)
}),
fetchTreeFiles: () =>
Effect.sync(() => {
Expand Down Expand Up @@ -575,7 +587,7 @@ describe("DependencyScanner", () => {
})

it.effect(
"fails a scan whose tree listing was cut short instead of fixing the unlisted lockfiles' findings",
"records a scan whose tree listing was cut short as failed instead of fixing the unlisted lockfiles' findings",
() => {
const state = world({ "package-lock.json": packageLock("4.17.15") })
return Effect.gen(function* () {
Expand All @@ -584,10 +596,11 @@ describe("DependencyScanner", () => {
state.head = sha("b")
state.truncated = true
state.files.delete("package-lock.json")
const error = yield* Effect.flip(
scanner.scan(installation, repository, { trigger: "schedule" }),
)
assert.include(error.message, "too large")
const outcome = yield* scanner.scan(installation, repository, { trigger: "schedule" })
assert.strictEqual(outcome._tag, "TreeTooLarge")
if (outcome._tag !== "TreeTooLarge") return
assert.deepStrictEqual([outcome.scan.status, outcome.scan.commitSha], ["failed", sha("b")])
assert.include(outcome.scan.error ?? "", "too large")
assert.isTrue(Arr.every(yield* findingsOf, (finding) => finding.status === "open"))
}).pipe(Effect.provide(scannerFor(state)))
},
Expand All @@ -599,17 +612,44 @@ describe("DependencyScanner", () => {
const scanner = yield* DependencyScanner
const store = yield* DependencyScanStore
state.truncated = true
yield* Effect.flip(scanner.scan(installation, repository, { trigger: "push" }))
const first = yield* scanner.scan(installation, repository, { trigger: "push" })
assert.strictEqual(first._tag, "TreeTooLarge")
state.head = sha("b")
const again = yield* scanner.scan(installation, repository, { trigger: "push" })
assert.strictEqual(again._tag, "Unchanged")
assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 1)
// Scan now records its own result, so the page sees the scan it asked for.
yield* Effect.flip(scanner.scan(installation, repository, { trigger: "manual" }))
yield* scanner.scan(installation, repository, { trigger: "manual" })
assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 2)
}).pipe(Effect.provide(scannerFor(state)))
})

it.effect("answers a repository the provider blocks or lost as unreachable, recording no scan", () => {
const state = world({ "package-lock.json": packageLock("4.17.15") })
const refusals = [
new VcsRepositoryBlockedError({ message: "Repository access blocked", status: 451 }),
new VcsRepoUnavailableError({ message: "Not Found" }),
new VcsInstallationGoneError({ message: "Not Found" }),
]
return Effect.gen(function* () {
const scanner = yield* DependencyScanner
const store = yield* DependencyScanStore
const reasons = yield* Effect.forEach(refusals, (refusal) => {
state.refusal = refusal
return scanner
.scan(installation, repository, { trigger: "schedule" })
.pipe(
Effect.map((outcome) =>
outcome._tag === "RepositoryUnreachable" ? outcome.reason : outcome._tag,
),
)
})
assert.deepStrictEqual(reasons, ["blocked", "unavailable", "installation_gone"])
assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 0)
assert.strictEqual(state.calls.fetchTreeFiles, 0)
}).pipe(Effect.provide(scannerFor(state)))
})

it.effect("reads a yarn lockfile's manifests but never one too large to be a package.json", () => {
const state = world({
"yarn.lock": '"lodash@4.17.15":\n version "4.17.15"\n',
Expand Down
Loading
Loading